Created
June 16, 2020 03:06
-
-
Save harshavardhana/2012fcce5fdcefd94f967e3b8661ebe1 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// +build ignore | |
/* | |
* MinIO Go Library for Amazon S3 Compatible Cloud Storage | |
* Copyright 2015-2017 MinIO, Inc. | |
* | |
* Licensed under the Apache License, Version 2.0 (the "License"); | |
* you may not use this file except in compliance with the License. | |
* You may obtain a copy of the License at | |
* | |
* http://www.apache.org/licenses/LICENSE-2.0 | |
* | |
* Unless required by applicable law or agreed to in writing, software | |
* distributed under the License is distributed on an "AS IS" BASIS, | |
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
* See the License for the specific language governing permissions and | |
* limitations under the License. | |
*/ | |
package main | |
import ( | |
"bytes" | |
"context" | |
"encoding/xml" | |
"errors" | |
"fmt" | |
"io" | |
"io/ioutil" | |
"math/rand" | |
"mime/multipart" | |
"net/http" | |
"net/url" | |
"os" | |
"path/filepath" | |
"reflect" | |
"runtime" | |
"sort" | |
"strconv" | |
"strings" | |
"time" | |
"github.com/dustin/go-humanize" | |
jsoniter "github.com/json-iterator/go" | |
log "github.com/sirupsen/logrus" | |
"github.com/minio/minio-go/v6" | |
"github.com/minio/minio-go/v6/pkg/encrypt" | |
) | |
const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" | |
const ( | |
letterIdxBits = 6 // 6 bits to represent a letter index | |
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits | |
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits | |
) | |
const ( | |
serverEndpoint = "SERVER_ENDPOINT" | |
accessKey = "ACCESS_KEY" | |
secretKey = "SECRET_KEY" | |
enableHTTPS = "ENABLE_HTTPS" | |
enableKMS = "ENABLE_KMS" | |
) | |
type mintJSONFormatter struct { | |
} | |
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { | |
data := make(log.Fields, len(entry.Data)) | |
for k, v := range entry.Data { | |
switch v := v.(type) { | |
case error: | |
// Otherwise errors are ignored by `encoding/json` | |
// https://github.com/sirupsen/logrus/issues/137 | |
data[k] = v.Error() | |
default: | |
data[k] = v | |
} | |
} | |
var json = jsoniter.ConfigCompatibleWithStandardLibrary | |
serialized, err := json.Marshal(data) | |
if err != nil { | |
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) | |
} | |
return append(serialized, '\n'), nil | |
} | |
func cleanEmptyEntries(fields log.Fields) log.Fields { | |
cleanFields := log.Fields{} | |
for k, v := range fields { | |
if v != "" { | |
cleanFields[k] = v | |
} | |
} | |
return cleanFields | |
} | |
// log successful test runs | |
func successLogger(testName string, function string, args map[string]interface{}, startTime time.Time) *log.Entry { | |
// calculate the test case duration | |
duration := time.Since(startTime) | |
// log with the fields as per mint | |
fields := log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": "PASS"} | |
return log.WithFields(cleanEmptyEntries(fields)) | |
} | |
// As few of the features are not available in Gateway(s) currently, Check if err value is NotImplemented, | |
// and log as NA in that case and continue execution. Otherwise log as failure and return | |
func logError(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) { | |
// If server returns NotImplemented we assume it is gateway mode and hence log it as info and move on to next tests | |
// Special case for ComposeObject API as it is implemented on client side and adds specific error details like `Error in upload-part-copy` in | |
// addition to NotImplemented error returned from server | |
if isErrNotImplemented(err) { | |
ignoredLog(testName, function, args, startTime, message).Info() | |
} else { | |
failureLog(testName, function, args, startTime, alert, message, err).Fatal() | |
} | |
} | |
// log failed test runs | |
func failureLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { | |
// calculate the test case duration | |
duration := time.Since(startTime) | |
var fields log.Fields | |
// log with the fields as per mint | |
if err != nil { | |
fields = log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, | |
"duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message, "error": err} | |
} else { | |
fields = log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, | |
"duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message} | |
} | |
return log.WithFields(cleanEmptyEntries(fields)) | |
} | |
// log not applicable test runs | |
func ignoredLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string) *log.Entry { | |
// calculate the test case duration | |
duration := time.Since(startTime) | |
// log with the fields as per mint | |
fields := log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, | |
"duration": duration.Nanoseconds() / 1000000, "status": "NA", "alert": alert} | |
return log.WithFields(cleanEmptyEntries(fields)) | |
} | |
// Delete objects in given bucket, recursively | |
func cleanupBucket(bucketName string, c *minio.Client) error { | |
// Create a done channel to control 'ListObjectsV2' go routine. | |
doneCh := make(chan struct{}) | |
// Exit cleanly upon return. | |
defer close(doneCh) | |
// Iterate over all objects in the bucket via listObjectsV2 and delete | |
for objCh := range c.ListObjectsV2(bucketName, "", true, doneCh) { | |
if objCh.Err != nil { | |
return objCh.Err | |
} | |
if objCh.Key != "" { | |
err := c.RemoveObject(bucketName, objCh.Key) | |
if err != nil { | |
return err | |
} | |
} | |
} | |
for objPartInfo := range c.ListIncompleteUploads(bucketName, "", true, doneCh) { | |
if objPartInfo.Err != nil { | |
return objPartInfo.Err | |
} | |
if objPartInfo.Key != "" { | |
err := c.RemoveIncompleteUpload(bucketName, objPartInfo.Key) | |
if err != nil { | |
return err | |
} | |
} | |
} | |
// objects are already deleted, clear the buckets now | |
err := c.RemoveBucket(bucketName) | |
if err != nil { | |
return err | |
} | |
return err | |
} | |
func cleanupVersionedBucket(bucketName string, c *minio.Client) error { | |
doneCh := make(chan struct{}) | |
defer close(doneCh) | |
for obj := range c.ListObjectVersions(bucketName, "", true, doneCh) { | |
if obj.Err != nil { | |
return obj.Err | |
} | |
if obj.Key != "" { | |
err := c.RemoveObjectWithOptions(bucketName, obj.Key, minio.RemoveObjectOptions{VersionID: obj.VersionID}) | |
if err != nil { | |
return err | |
} | |
} | |
} | |
for objPartInfo := range c.ListIncompleteUploads(bucketName, "", true, doneCh) { | |
if objPartInfo.Err != nil { | |
return objPartInfo.Err | |
} | |
if objPartInfo.Key != "" { | |
err := c.RemoveIncompleteUpload(bucketName, objPartInfo.Key) | |
if err != nil { | |
return err | |
} | |
} | |
} | |
// objects are already deleted, clear the buckets now | |
err := c.RemoveBucket(bucketName) | |
if err != nil { | |
return err | |
} | |
return err | |
} | |
func isErrNotImplemented(err error) bool { | |
return minio.ToErrorResponse(err).Code == "NotImplemented" | |
} | |
func init() { | |
// If server endpoint is not set, all tests default to | |
// using https://play.min.io | |
if os.Getenv(serverEndpoint) == "" { | |
os.Setenv(serverEndpoint, "play.min.io") | |
os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F") | |
os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") | |
os.Setenv(enableHTTPS, "1") | |
} | |
} | |
var mintDataDir = os.Getenv("MINT_DATA_DIR") | |
func getMintDataDirFilePath(filename string) (fp string) { | |
if mintDataDir == "" { | |
return | |
} | |
return filepath.Join(mintDataDir, filename) | |
} | |
type sizedReader struct { | |
io.Reader | |
size int | |
} | |
func (l *sizedReader) Size() int { | |
return l.size | |
} | |
func (l *sizedReader) Close() error { | |
return nil | |
} | |
type randomReader struct{ seed []byte } | |
func (r *randomReader) Read(b []byte) (int, error) { | |
return copy(b, bytes.Repeat(r.seed, len(b))), nil | |
} | |
// read data from file if it exists or optionally create a buffer of particular size | |
func getDataReader(fileName string) io.ReadCloser { | |
if mintDataDir == "" { | |
size := dataFileMap[fileName] | |
return &sizedReader{ | |
Reader: io.LimitReader(&randomReader{ | |
seed: []byte("a"), | |
}, int64(size)), | |
size: size, | |
} | |
} | |
reader, _ := os.Open(getMintDataDirFilePath(fileName)) | |
return reader | |
} | |
// randString generates random names and prepends them with a known prefix. | |
func randString(n int, src rand.Source, prefix string) string { | |
b := make([]byte, n) | |
// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters! | |
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; { | |
if remain == 0 { | |
cache, remain = src.Int63(), letterIdxMax | |
} | |
if idx := int(cache & letterIdxMask); idx < len(letterBytes) { | |
b[i] = letterBytes[idx] | |
i-- | |
} | |
cache >>= letterIdxBits | |
remain-- | |
} | |
return prefix + string(b[0:30-len(prefix)]) | |
} | |
var dataFileMap = map[string]int{ | |
"datafile-1-b": 1, | |
"datafile-10-kB": 10 * humanize.KiByte, | |
"datafile-33-kB": 33 * humanize.KiByte, | |
"datafile-100-kB": 100 * humanize.KiByte, | |
"datafile-1.03-MB": 1056 * humanize.KiByte, | |
"datafile-1-MB": 1 * humanize.MiByte, | |
"datafile-5-MB": 5 * humanize.MiByte, | |
"datafile-6-MB": 6 * humanize.MiByte, | |
"datafile-11-MB": 11 * humanize.MiByte, | |
"datafile-129-MB": 129 * humanize.MiByte, | |
} | |
func isFullMode() bool { | |
return os.Getenv("MINT_MODE") == "full" | |
} | |
func getFuncName() string { | |
pc, _, _, _ := runtime.Caller(1) | |
return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.") | |
} | |
// Tests bucket re-create errors. | |
func testMakeBucketError() { | |
region := "eu-central-1" | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "MakeBucket(bucketName, region)" | |
// initialize logging params | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"region": region, | |
} | |
// skipping region functional tests for non s3 runs | |
if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { | |
ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() | |
return | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket in 'eu-central-1'. | |
if err = c.MakeBucket(bucketName, region); err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket Failed", err) | |
return | |
} | |
if err = c.MakeBucket(bucketName, region); err == nil { | |
logError(testName, function, args, startTime, "", "Bucket already exists", err) | |
return | |
} | |
// Verify valid error response from server. | |
if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && | |
minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { | |
logError(testName, function, args, startTime, "", "Invalid error returned by server", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testMetadataSizeLimit() { | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, reader, objectSize, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"opts.UserMetadata": "", | |
} | |
rand.Seed(startTime.Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client creation failed", err) | |
return | |
} | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
const HeaderSizeLimit = 8 * 1024 | |
const UserMetadataLimit = 2 * 1024 | |
// Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail | |
metadata := make(map[string]string) | |
metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test"))) | |
args["metadata"] = fmt.Sprint(metadata) | |
_, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil) | |
return | |
} | |
// Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail | |
metadata = make(map[string]string) | |
metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test"))) | |
args["metadata"] = fmt.Sprint(metadata) | |
_, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests various bucket supported formats. | |
func testMakeBucketRegions() { | |
region := "eu-central-1" | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "MakeBucket(bucketName, region)" | |
// initialize logging params | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"region": region, | |
} | |
// skipping region functional tests for non s3 runs | |
if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { | |
ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() | |
return | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket in 'eu-central-1'. | |
if err = c.MakeBucket(bucketName, region); err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
// Make a new bucket with '.' in its name, in 'us-west-2'. This | |
// request is internally staged into a path style instead of | |
// virtual host style. | |
region = "us-west-2" | |
args["region"] = region | |
if err = c.MakeBucket(bucketName+".withperiod", region); err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName+".withperiod", c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test PutObject using a large data to trigger multipart readat | |
func testPutObjectReadAt() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, reader, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"opts": "objectContentType", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
bufSize := dataFileMap["datafile-129-MB"] | |
var reader = getDataReader("datafile-129-MB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Object content type | |
objectContentType := "binary/octet-stream" | |
args["objectContentType"] = objectContentType | |
n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Get Object failed", err) | |
return | |
} | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat Object failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err) | |
return | |
} | |
if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "Content types don't match", err) | |
return | |
} | |
if err := r.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "Object Close failed", err) | |
return | |
} | |
if err := r.Close(); err == nil { | |
logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testListObjectVersions() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "ListObjectVersions(bucketName, prefix, recursive)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"prefix": "", | |
"recursive": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
err = c.EnableVersioning(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Enable versioning failed", err) | |
return | |
} | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
bufSize := dataFileMap["datafile-10-kB"] | |
var reader = getDataReader("datafile-10-kB") | |
n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) | |
return | |
} | |
reader.Close() | |
bufSize = dataFileMap["datafile-1-b"] | |
reader = getDataReader("datafile-1-b") | |
n, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) | |
return | |
} | |
reader.Close() | |
err = c.RemoveObject(bucketName, objectName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected object deletion", err) | |
return | |
} | |
var deleteMarkers, versions int | |
doneCh := make(chan struct{}) | |
objectsInfo := c.ListObjectVersions(bucketName, "", true, doneCh) | |
for info := range objectsInfo { | |
if info.Err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) | |
return | |
} | |
if info.Key != objectName { | |
logError(testName, function, args, startTime, "", "Unexpected object name in listing objects", nil) | |
return | |
} | |
if info.VersionID == "" { | |
logError(testName, function, args, startTime, "", "Unexpected version id in listing objects", nil) | |
return | |
} | |
if info.IsDeleteMarker { | |
deleteMarkers++ | |
if !info.IsLatest { | |
logError(testName, function, args, startTime, "", "Unexpected IsLatest field in listing objects", nil) | |
return | |
} | |
} else { | |
versions++ | |
} | |
} | |
if deleteMarkers != 1 { | |
logError(testName, function, args, startTime, "", "Unexpected number of DeleteMarker elements in listing objects", nil) | |
return | |
} | |
if versions != 2 { | |
logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) | |
return | |
} | |
// Delete all objects and their versions as long as the bucket itself | |
if err = cleanupVersionedBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testStatObjectWithVersioning() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "StatObject" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
err = c.EnableVersioning(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Enable versioning failed", err) | |
return | |
} | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
bufSize := dataFileMap["datafile-10-kB"] | |
var reader = getDataReader("datafile-10-kB") | |
n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) | |
return | |
} | |
reader.Close() | |
bufSize = dataFileMap["datafile-1-b"] | |
reader = getDataReader("datafile-1-b") | |
n, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) | |
return | |
} | |
reader.Close() | |
doneCh := make(chan struct{}) | |
objectsInfo := c.ListObjectVersions(bucketName, "", true, doneCh) | |
var results []minio.ObjectVersionInfo | |
for info := range objectsInfo { | |
if info.Err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) | |
return | |
} | |
results = append(results, info) | |
} | |
if len(results) != 2 { | |
logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) | |
return | |
} | |
for i := 0; i < len(results); i++ { | |
opts := minio.StatObjectOptions{minio.GetObjectOptions{VersionID: results[i].VersionID}} | |
statInfo, err := c.StatObject(bucketName, objectName, opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "error during HEAD object", err) | |
return | |
} | |
if statInfo.ETag != results[i].ETag { | |
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) | |
return | |
} | |
// We don't care about nano-second precision | |
if results[i].LastModified.Unix() != statInfo.LastModified.Unix() { | |
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) | |
return | |
} | |
if statInfo.Size != results[i].Size { | |
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) | |
return | |
} | |
} | |
// Delete all objects and their versions as long as the bucket itself | |
if err = cleanupVersionedBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testGetObjectWithVersioning() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject()" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
err = c.EnableVersioning(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Enable versioning failed", err) | |
return | |
} | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Save the contents of datafiles to check with GetObject() reader output later | |
var buffers [][]byte | |
var testFiles = []string{"datafile-1-b", "datafile-10-kB"} | |
for _, testFile := range testFiles { | |
r := getDataReader(testFile) | |
buf, err := ioutil.ReadAll(r) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "unexpected failure", err) | |
return | |
} | |
r.Close() | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", | |
"Number of bytes returned by PutObject does not match, expected "+string(len(buf))+" got "+string(n), err) | |
return | |
} | |
buffers = append(buffers, buf) | |
} | |
doneCh := make(chan struct{}) | |
objectsInfo := c.ListObjectVersions(bucketName, "", true, doneCh) | |
var results []minio.ObjectVersionInfo | |
for info := range objectsInfo { | |
if info.Err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) | |
return | |
} | |
results = append(results, info) | |
} | |
if len(results) != 2 { | |
logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) | |
return | |
} | |
sort.SliceStable(results, func(i, j int) bool { | |
return results[i].Size < results[j].Size | |
}) | |
sort.SliceStable(buffers, func(i, j int) bool { | |
return len(buffers[i]) < len(buffers[j]) | |
}) | |
for i := 0; i < len(results); i++ { | |
opts := minio.GetObjectOptions{VersionID: results[i].VersionID} | |
reader, err := c.GetObject(bucketName, objectName, opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "error during GET object", err) | |
return | |
} | |
statInfo, err := reader.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err) | |
return | |
} | |
if statInfo.ETag != results[i].ETag { | |
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) | |
return | |
} | |
// We don't care about nano-second precision | |
if results[i].LastModified.Unix() != statInfo.LastModified.Unix() { | |
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) | |
return | |
} | |
if statInfo.Size != results[i].Size { | |
logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) | |
return | |
} | |
tmpBuffer := bytes.NewBuffer([]byte{}) | |
_, err = io.Copy(tmpBuffer, reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "unexpected io.Copy()", err) | |
return | |
} | |
if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) { | |
fmt.Println(len(tmpBuffer.Bytes()), len(buffers[i])) | |
logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err) | |
return | |
} | |
} | |
// Delete all objects and their versions as long as the bucket itself | |
if err = cleanupVersionedBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testCopyObjectWithVersioning() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject()" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
err = c.EnableVersioning(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Enable versioning failed", err) | |
return | |
} | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
var testFiles = []string{"datafile-1-b", "datafile-10-kB"} | |
for _, testFile := range testFiles { | |
r := getDataReader(testFile) | |
buf, err := ioutil.ReadAll(r) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "unexpected failure", err) | |
return | |
} | |
r.Close() | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", | |
"Number of bytes returned by PutObject does not match, expected "+string(len(buf))+" got "+string(n), err) | |
return | |
} | |
} | |
doneCh := make(chan struct{}) | |
objectsVersionsInfo := c.ListObjectVersions(bucketName, "", true, doneCh) | |
var infos []minio.ObjectVersionInfo | |
for info := range objectsVersionsInfo { | |
if info.Err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) | |
return | |
} | |
infos = append(infos, info) | |
} | |
sort.Slice(infos, func(i, j int) bool { | |
return infos[i].Size < infos[j].Size | |
}) | |
reader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err) | |
return | |
} | |
oldestContent, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err) | |
return | |
} | |
// Copy Source | |
src := minio.NewSourceInfo(bucketName, objectName, nil) | |
src.SetVersionID(infos[0].VersionID) | |
args["src"] = src | |
dst, err := minio.NewDestinationInfo(bucketName, objectName+"-copy", nil, nil) | |
args["dst"] = dst | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
// Perform the Copy | |
err = c.CopyObject(dst, src) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed", err) | |
return | |
} | |
// Destination object | |
readerCopy, err := c.GetObject(bucketName, objectName+"-copy", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
defer readerCopy.Close() | |
newestContent, err := ioutil.ReadAll(readerCopy) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err) | |
return | |
} | |
if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) { | |
logError(testName, function, args, startTime, "", "Unexpected destination object content", err) | |
return | |
} | |
// Delete all objects and their versions as long as the bucket itself | |
if err = cleanupVersionedBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testComposeObjectWithVersioning() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "ComposeObject()" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
err = c.EnableVersioning(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Enable versioning failed", err) | |
return | |
} | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// var testFiles = []string{"datafile-5-MB", "datafile-10-kB"} | |
var testFiles = []string{"datafile-5-MB", "datafile-10-kB"} | |
var testFilesBytes [][]byte | |
for _, testFile := range testFiles { | |
r := getDataReader(testFile) | |
buf, err := ioutil.ReadAll(r) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "unexpected failure", err) | |
return | |
} | |
r.Close() | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", | |
"Number of bytes returned by PutObject does not match, expected "+string(len(buf))+" got "+string(n), err) | |
return | |
} | |
testFilesBytes = append(testFilesBytes, buf) | |
} | |
doneCh := make(chan struct{}) | |
objectsInfo := c.ListObjectVersions(bucketName, "", true, doneCh) | |
var results []minio.ObjectVersionInfo | |
for info := range objectsInfo { | |
if info.Err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) | |
return | |
} | |
results = append(results, info) | |
} | |
sort.SliceStable(results, func(i, j int) bool { | |
return results[i].Size > results[j].Size | |
}) | |
// Source objects to concatenate. We also specify decryption | |
// key for each | |
src1 := minio.NewSourceInfo(bucketName, objectName, nil) | |
src1.SetVersionID(results[0].VersionID) | |
src2 := minio.NewSourceInfo(bucketName, objectName, nil) | |
src2.SetVersionID(results[1].VersionID) | |
// Create slice of sources. | |
srcs := []minio.SourceInfo{src1, src2} | |
// Create destination info | |
dst, err := minio.NewDestinationInfo(bucketName, objectName+"-copy", nil, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
err = c.ComposeObject(dst, srcs) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ComposeObject failed", err) | |
return | |
} | |
// Destination object | |
readerCopy, err := c.GetObject(bucketName, objectName+"-copy", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject of the copy object failed", err) | |
return | |
} | |
defer readerCopy.Close() | |
copyContentBytes, err := ioutil.ReadAll(readerCopy) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Reading from the copy object reader failed", err) | |
return | |
} | |
var expectedContent []byte | |
for _, fileBytes := range testFilesBytes { | |
expectedContent = append(expectedContent, fileBytes...) | |
} | |
if len(copyContentBytes) == 0 || !bytes.Equal(copyContentBytes, expectedContent) { | |
logError(testName, function, args, startTime, "", "Unexpected destination object content", err) | |
return | |
} | |
// Delete all objects and their versions as long as the bucket itself | |
if err = cleanupVersionedBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testRemoveObjectWithVersioning() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "DeleteObject()" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
err = c.EnableVersioning(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Enable versioning failed", err) | |
return | |
} | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
n, err := c.PutObject(bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(dataFileMap["datafile-10-kB"]) { | |
logError(testName, function, args, startTime, "", | |
"Number of bytes returned by PutObject does not match, expected "+string(dataFileMap["datafile-10-kB"])+" got "+string(n), err) | |
return | |
} | |
doneCh := make(chan struct{}) | |
objectsInfo := c.ListObjectVersions(bucketName, "", true, doneCh) | |
var version minio.ObjectVersionInfo | |
for info := range objectsInfo { | |
if info.Err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) | |
return | |
} | |
version = info | |
break | |
} | |
err = c.RemoveObjectWithOptions(bucketName, objectName, minio.RemoveObjectOptions{VersionID: version.VersionID}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "DeleteObject failed", err) | |
return | |
} | |
doneCh = make(chan struct{}) | |
objectsInfo = c.ListObjectVersions(bucketName, "", true, doneCh) | |
for range objectsInfo { | |
logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err) | |
return | |
} | |
err = c.RemoveBucket(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testRemoveObjectsWithVersioning() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "DeleteObjects()" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
err = c.EnableVersioning(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Enable versioning failed", err) | |
return | |
} | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
n, err := c.PutObject(bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(dataFileMap["datafile-10-kB"]) { | |
logError(testName, function, args, startTime, "", | |
"Number of bytes returned by PutObject does not match, expected "+string(dataFileMap["datafile-10-kB"])+" got "+string(n), err) | |
return | |
} | |
objectsVersions := make(chan minio.ObjectVersion) | |
go func() { | |
doneCh := make(chan struct{}) | |
objectsVersionsInfo := c.ListObjectVersions(bucketName, "", true, doneCh) | |
for info := range objectsVersionsInfo { | |
if info.Err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) | |
return | |
} | |
objectsVersions <- minio.ObjectVersion{ | |
Key: info.Key, | |
VersionID: info.VersionID, | |
} | |
} | |
close(objectsVersions) | |
}() | |
removeErrors := c.RemoveObjectsWithVersions(context.Background(), bucketName, objectsVersions, minio.RemoveObjectsOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "DeleteObjects call failed", err) | |
return | |
} | |
for e := range removeErrors { | |
if e.Err != nil { | |
logError(testName, function, args, startTime, "", "Single delete operation failed", err) | |
return | |
} | |
} | |
doneCh := make(chan struct{}) | |
objectsVersionsInfo := c.ListObjectVersions(bucketName, "", true, doneCh) | |
for range objectsVersionsInfo { | |
logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err) | |
return | |
} | |
err = c.RemoveBucket(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testObjectTaggingWithVersioning() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "{Get,Set,Remove}ObjectTagging()" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
err = c.EnableVersioning(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Enable versioning failed", err) | |
return | |
} | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
for _, file := range []string{"datafile-1-b", "datafile-10-kB"} { | |
n, err := c.PutObject(bucketName, objectName, getDataReader(file), int64(dataFileMap[file]), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(dataFileMap[file]) { | |
logError(testName, function, args, startTime, "", | |
"Number of bytes returned by PutObject does not match, expected "+string(dataFileMap[file])+" got "+string(n), err) | |
return | |
} | |
} | |
doneCh := make(chan struct{}) | |
versionsInfo := c.ListObjectVersions(bucketName, "", true, doneCh) | |
var versions []minio.ObjectVersionInfo | |
for info := range versionsInfo { | |
if info.Err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) | |
return | |
} | |
versions = append(versions, info) | |
} | |
sort.SliceStable(versions, func(i, j int) bool { | |
return versions[i].Size < versions[j].Size | |
}) | |
tagsV1 := map[string]string{"key1": "val1"} | |
err = c.PutObjectTaggingWithOptions(context.Background(), bucketName, objectName, tagsV1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectTaggingWithOptions (1) failed", err) | |
return | |
} | |
tagsV2 := map[string]string{"key2": "val2"} | |
err = c.PutObjectTaggingWithOptions(context.Background(), bucketName, objectName, tagsV2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectTaggingWithOptions (2) failed", err) | |
return | |
} | |
type Tag struct { | |
Key string | |
Value string | |
} | |
type TagSet struct { | |
Tag []Tag | |
} | |
type GetObjectTaggingOutput struct { | |
TagSet TagSet | |
} | |
tagsEqual := func(tags map[string]string, getObjectTagging GetObjectTaggingOutput) bool { | |
for k, v := range tags { | |
found := false | |
for _, t := range getObjectTagging.TagSet.Tag { | |
if k+"="+v == t.Key+"="+t.Value { | |
found = true | |
break | |
} | |
} | |
if !found { | |
return false | |
} | |
} | |
return true | |
} | |
tagsV1XML, err := c.GetObjectTaggingWithOptions(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObjectTaggingWithOptions failed", err) | |
return | |
} | |
var getObjectTaggingV1 GetObjectTaggingOutput | |
err = xml.Unmarshal([]byte(tagsV1XML), &getObjectTaggingV1) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during XML unmarshal (1)", err) | |
return | |
} | |
if !tagsEqual(tagsV1, getObjectTaggingV1) { | |
logError(testName, function, args, startTime, "", "Unexpected tags content (1)", err) | |
return | |
} | |
tagsV2XML, err := c.GetObjectTaggingWithContext(context.Background(), bucketName, objectName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObjectTaggingWithContext failed", err) | |
return | |
} | |
var getObjectTaggingV2 GetObjectTaggingOutput | |
err = xml.Unmarshal([]byte(tagsV2XML), &getObjectTaggingV2) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during XML unmarshal (2)", err) | |
return | |
} | |
if !tagsEqual(tagsV2, getObjectTaggingV2) { | |
logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) | |
return | |
} | |
err = c.RemoveObjectTaggingWithOptions(context.Background(), bucketName, objectName, minio.RemoveObjectTaggingOptions{VersionID: versions[0].VersionID}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectTaggingWithOptions (2) failed", err) | |
return | |
} | |
emptyTagsXML, err := c.GetObjectTaggingWithOptions(context.Background(), bucketName, objectName, | |
minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObjectTaggingWithOptions failed", err) | |
return | |
} | |
var getObjectExpectedEmptyTagging GetObjectTaggingOutput | |
err = xml.Unmarshal([]byte(emptyTagsXML), &getObjectExpectedEmptyTagging) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error during XML unmarshal (3)", err) | |
return | |
} | |
if len(getObjectExpectedEmptyTagging.TagSet.Tag) != 0 { | |
logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) | |
return | |
} | |
// Delete all objects and their versions as long as the bucket itself | |
if err = cleanupVersionedBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test PutObject using a large data to trigger multipart readat | |
func testPutObjectWithMetadata() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, reader,size, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", | |
} | |
if !isFullMode() { | |
ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info() | |
return | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Make bucket failed", err) | |
return | |
} | |
bufSize := dataFileMap["datafile-129-MB"] | |
var reader = getDataReader("datafile-129-MB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Object custom metadata | |
customContentType := "custom/contenttype" | |
args["metadata"] = map[string][]string{ | |
"Content-Type": {customContentType}, | |
"X-Amz-Meta-CustomKey": {"extra spaces in value"}, | |
} | |
n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ | |
ContentType: customContentType}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) | |
return | |
} | |
if st.ContentType != customContentType && st.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err) | |
return | |
} | |
if err := r.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "Object Close failed", err) | |
return | |
} | |
if err := r.Close(); err == nil { | |
logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testPutObjectWithContentLanguage() { | |
// initialize logging params | |
objectName := "test-object" | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, reader, size, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": objectName, | |
"size": -1, | |
"opts": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
data := bytes.Repeat([]byte("a"), int(0)) | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ | |
ContentLanguage: "en", | |
}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != 0 { | |
logError(testName, function, args, startTime, "", "Expected upload object '0' doesn't match with PutObject return value", err) | |
return | |
} | |
objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
if objInfo.Metadata.Get("Content-Language") != "en" { | |
logError(testName, function, args, startTime, "", "Expected content-language 'en' doesn't match with StatObject return value", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test put object with streaming signature. | |
func testPutObjectStreaming() { | |
// initialize logging params | |
objectName := "test-object" | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, reader,size,opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": objectName, | |
"size": -1, | |
"opts": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Upload an object. | |
sizes := []int64{0, 64*1024 - 1, 64 * 1024} | |
for _, size := range sizes { | |
data := bytes.Repeat([]byte("a"), int(size)) | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) | |
return | |
} | |
if n != size { | |
logError(testName, function, args, startTime, "", "Expected upload object size doesn't match with PutObjectStreaming return value", err) | |
return | |
} | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test get object seeker from the end, using whence set to '2'. | |
func testGetObjectSeekEnd() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) | |
return | |
} | |
pos, err := r.Seek(-100, 2) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Object Seek failed", err) | |
return | |
} | |
if pos != st.Size-100 { | |
logError(testName, function, args, startTime, "", "Incorrect position", err) | |
return | |
} | |
buf2 := make([]byte, 100) | |
m, err := io.ReadFull(r, buf2) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Error reading through io.ReadFull", err) | |
return | |
} | |
if m != len(buf2) { | |
logError(testName, function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err) | |
return | |
} | |
hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) | |
hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) | |
if hexBuf1 != hexBuf2 { | |
logError(testName, function, args, startTime, "", "Values at same index dont match", err) | |
return | |
} | |
pos, err = r.Seek(-100, 2) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Object Seek failed", err) | |
return | |
} | |
if pos != st.Size-100 { | |
logError(testName, function, args, startTime, "", "Incorrect position", err) | |
return | |
} | |
if err = r.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "ObjectClose failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test get object reader to not throw error on being closed twice. | |
func testGetObjectClosedTwice() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "PutObject response doesn't match sent bytes, expected "+string(int64(bufSize))+" got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) | |
return | |
} | |
if err := r.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "Object Close failed", err) | |
return | |
} | |
if err := r.Close(); err == nil { | |
logError(testName, function, args, startTime, "", "Already closed object. No error returned", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test RemoveObjectsWithContext request context cancels after timeout | |
func testRemoveObjectsWithContext() { | |
// Initialize logging params. | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "RemoveObjectsWithContext(ctx, bucketName, objectsCh)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
} | |
// Seed random based on current tie. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Enable tracing, write to stdout. | |
// c.TraceOn(os.Stderr) | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
// Generate put data. | |
r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) | |
// Multi remove of 20 objects. | |
nrObjects := 20 | |
objectsCh := make(chan string) | |
go func() { | |
defer close(objectsCh) | |
for i := 0; i < nrObjects; i++ { | |
objectName := "sample" + strconv.Itoa(i) + ".txt" | |
_, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
continue | |
} | |
objectsCh <- objectName | |
} | |
}() | |
// Set context to cancel in 1 nanosecond. | |
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) | |
args["ctx"] = ctx | |
defer cancel() | |
// Call RemoveObjectsWithContext API with short timeout. | |
errorCh := c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) | |
// Check for error. | |
select { | |
case r := <-errorCh: | |
if r.Err == nil { | |
logError(testName, function, args, startTime, "", "RemoveObjectsWithContext should fail on short timeout", err) | |
return | |
} | |
} | |
// Set context with longer timeout. | |
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) | |
args["ctx"] = ctx | |
defer cancel() | |
// Perform RemoveObjectsWithContext with the longer timeout. Expect the removals to succeed. | |
errorCh = c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) | |
select { | |
case r, more := <-errorCh: | |
if more || r.Err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error", r.Err) | |
return | |
} | |
} | |
// Delete all objects and buckets. | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test removing multiple objects with Remove API | |
func testRemoveMultipleObjects() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "RemoveObjects(bucketName, objectsCh)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Enable tracing, write to stdout. | |
// c.TraceOn(os.Stderr) | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) | |
// Multi remove of 1100 objects | |
nrObjects := 200 | |
objectsCh := make(chan string) | |
go func() { | |
defer close(objectsCh) | |
// Upload objects and send them to objectsCh | |
for i := 0; i < nrObjects; i++ { | |
objectName := "sample" + strconv.Itoa(i) + ".txt" | |
_, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
continue | |
} | |
objectsCh <- objectName | |
} | |
}() | |
// Call RemoveObjects API | |
errorCh := c.RemoveObjects(bucketName, objectsCh) | |
// Check if errorCh doesn't receive any error | |
select { | |
case r, more := <-errorCh: | |
if more { | |
logError(testName, function, args, startTime, "", "Unexpected error", r.Err) | |
return | |
} | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests FPutObject of a big file to trigger multipart | |
func testFPutObjectMultipart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "FPutObject(bucketName, objectName, fileName, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"fileName": "", | |
"opts": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. | |
var fileName = getMintDataDirFilePath("datafile-129-MB") | |
if fileName == "" { | |
// Make a temp file with minPartSize bytes of data. | |
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "TempFile creation failed", err) | |
return | |
} | |
// Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload. | |
if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { | |
logError(testName, function, args, startTime, "", "Copy failed", err) | |
return | |
} | |
if err = file.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "File Close failed", err) | |
return | |
} | |
fileName = file.Name() | |
args["fileName"] = fileName | |
} | |
totalSize := dataFileMap["datafile-129-MB"] | |
// Set base object name | |
objectName := bucketName + "FPutObject" + "-standard" | |
args["objectName"] = objectName | |
objectContentType := "testapplication/octet-stream" | |
args["objectContentType"] = objectContentType | |
// Perform standard FPutObject with contentType provided (Expecting application/octet-stream) | |
n, err := c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FPutObject failed", err) | |
return | |
} | |
if n != int64(totalSize) { | |
logError(testName, function, args, startTime, "", "FPutObject failed", err) | |
return | |
} | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
objInfo, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Unexpected error", err) | |
return | |
} | |
if objInfo.Size != int64(totalSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err) | |
return | |
} | |
if objInfo.ContentType != objectContentType && objInfo.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "ContentType doesn't match", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests FPutObject with null contentType (default = application/octet-stream) | |
func testFPutObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "FPutObject(bucketName, objectName, fileName, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"fileName": "", | |
"opts": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
location := "us-east-1" | |
// Make a new bucket. | |
args["bucketName"] = bucketName | |
args["location"] = location | |
function = "MakeBucket()bucketName, location" | |
err = c.MakeBucket(bucketName, location) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. | |
// Use different data in part for multipart tests to check parts are uploaded in correct order. | |
var fName = getMintDataDirFilePath("datafile-129-MB") | |
if fName == "" { | |
// Make a temp file with minPartSize bytes of data. | |
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "TempFile creation failed", err) | |
return | |
} | |
// Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload. | |
if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { | |
logError(testName, function, args, startTime, "", "File copy failed", err) | |
return | |
} | |
// Close the file pro-actively for windows. | |
if err = file.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "File close failed", err) | |
return | |
} | |
defer os.Remove(file.Name()) | |
fName = file.Name() | |
} | |
totalSize := dataFileMap["datafile-129-MB"] | |
// Set base object name | |
function = "FPutObject(bucketName, objectName, fileName, opts)" | |
objectName := bucketName + "FPutObject" | |
args["objectName"] = objectName + "-standard" | |
args["fileName"] = fName | |
args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"} | |
// Perform standard FPutObject with contentType provided (Expecting application/octet-stream) | |
n, err := c.FPutObject(bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FPutObject failed", err) | |
return | |
} | |
if n != int64(totalSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) | |
return | |
} | |
// Perform FPutObject with no contentType provided (Expecting application/octet-stream) | |
args["objectName"] = objectName + "-Octet" | |
n, err = c.FPutObject(bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "File close failed", err) | |
return | |
} | |
if n != int64(totalSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) | |
return | |
} | |
srcFile, err := os.Open(fName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "File open failed", err) | |
return | |
} | |
defer srcFile.Close() | |
// Add extension to temp file name | |
tmpFile, err := os.Create(fName + ".gtar") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "File create failed", err) | |
return | |
} | |
defer tmpFile.Close() | |
_, err = io.Copy(tmpFile, srcFile) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "File copy failed", err) | |
return | |
} | |
// Perform FPutObject with no contentType provided (Expecting application/x-gtar) | |
args["objectName"] = objectName + "-GTar" | |
args["opts"] = minio.PutObjectOptions{} | |
n, err = c.FPutObject(bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FPutObject failed", err) | |
return | |
} | |
if n != int64(totalSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) | |
return | |
} | |
// Check headers | |
function = "StatObject(bucketName, objectName, opts)" | |
args["objectName"] = objectName + "-standard" | |
rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
if rStandard.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err) | |
return | |
} | |
function = "StatObject(bucketName, objectName, opts)" | |
args["objectName"] = objectName + "-Octet" | |
rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
if rOctet.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err) | |
return | |
} | |
function = "StatObject(bucketName, objectName, opts)" | |
args["objectName"] = objectName + "-GTar" | |
rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-gtar or application/octet-stream, got "+rGTar.ContentType, err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
os.Remove(fName + ".gtar") | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests FPutObjectWithContext request context cancels after timeout | |
func testFPutObjectWithContext() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "FPutObject(bucketName, objectName, fileName, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"fileName": "", | |
"opts": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Upload 1 parts worth of data to use multipart upload. | |
// Use different data in part for multipart tests to check parts are uploaded in correct order. | |
var fName = getMintDataDirFilePath("datafile-1-MB") | |
if fName == "" { | |
// Make a temp file with 1 MiB bytes of data. | |
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "TempFile creation failed", err) | |
return | |
} | |
// Upload 1 parts to trigger multipart upload | |
if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { | |
logError(testName, function, args, startTime, "", "File copy failed", err) | |
return | |
} | |
// Close the file pro-actively for windows. | |
if err = file.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "File close failed", err) | |
return | |
} | |
defer os.Remove(file.Name()) | |
fName = file.Name() | |
} | |
totalSize := dataFileMap["datafile-1-MB"] | |
// Set base object name | |
objectName := bucketName + "FPutObjectWithContext" | |
args["objectName"] = objectName | |
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) | |
args["ctx"] = ctx | |
defer cancel() | |
// Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) | |
_, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) | |
return | |
} | |
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) | |
defer cancel() | |
// Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed | |
n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on long timeout", err) | |
return | |
} | |
if n != int64(totalSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) | |
return | |
} | |
_, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests FPutObjectWithContext request context cancels after timeout | |
func testFPutObjectWithContextV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "FPutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"opts": "minio.PutObjectOptions{ContentType:objectContentType}", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Upload 1 parts worth of data to use multipart upload. | |
// Use different data in part for multipart tests to check parts are uploaded in correct order. | |
var fName = getMintDataDirFilePath("datafile-1-MB") | |
if fName == "" { | |
// Make a temp file with 1 MiB bytes of data. | |
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Temp file creation failed", err) | |
return | |
} | |
// Upload 1 parts to trigger multipart upload | |
if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { | |
logError(testName, function, args, startTime, "", "File copy failed", err) | |
return | |
} | |
// Close the file pro-actively for windows. | |
if err = file.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "File close failed", err) | |
return | |
} | |
defer os.Remove(file.Name()) | |
fName = file.Name() | |
} | |
totalSize := dataFileMap["datafile-1-MB"] | |
// Set base object name | |
objectName := bucketName + "FPutObjectWithContext" | |
args["objectName"] = objectName | |
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) | |
args["ctx"] = ctx | |
defer cancel() | |
// Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) | |
_, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) | |
return | |
} | |
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) | |
defer cancel() | |
// Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed | |
n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on longer timeout", err) | |
return | |
} | |
if n != int64(totalSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match:wanted"+string(totalSize)+" got "+string(n), err) | |
return | |
} | |
_, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test validates putObject with context to see if request cancellation is honored. | |
func testPutObjectWithContext() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" | |
args := map[string]interface{}{ | |
"ctx": "", | |
"bucketName": "", | |
"objectName": "", | |
"opts": "", | |
} | |
// Instantiate new minio client object. | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Make a new bucket. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket call failed", err) | |
return | |
} | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) | |
args["objectName"] = objectName | |
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) | |
args["ctx"] = ctx | |
args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"} | |
defer cancel() | |
_, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "PutObjectWithContext should fail on short timeout", err) | |
return | |
} | |
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) | |
args["ctx"] = ctx | |
defer cancel() | |
reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
_, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests get object ReaderSeeker interface methods. | |
func testGetObjectReadSeekFunctional() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
defer func() { | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
}() | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat object failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) | |
return | |
} | |
// This following function helps us to compare data from the reader after seek | |
// with the data from the original buffer | |
cmpData := func(r io.Reader, start, end int) { | |
if end-start == 0 { | |
return | |
} | |
buffer := bytes.NewBuffer([]byte{}) | |
if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "CopyN failed", err) | |
return | |
} | |
} | |
if !bytes.Equal(buf[start:end], buffer.Bytes()) { | |
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) | |
return | |
} | |
} | |
// Generic seek error for errors other than io.EOF | |
seekErr := errors.New("seek error") | |
testCases := []struct { | |
offset int64 | |
whence int | |
pos int64 | |
err error | |
shouldCmp bool | |
start int | |
end int | |
}{ | |
// Start from offset 0, fetch data and compare | |
{0, 0, 0, nil, true, 0, 0}, | |
// Start from offset 2048, fetch data and compare | |
{2048, 0, 2048, nil, true, 2048, bufSize}, | |
// Start from offset larger than possible | |
{int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, | |
// Move to offset 0 without comparing | |
{0, 0, 0, nil, false, 0, 0}, | |
// Move one step forward and compare | |
{1, 1, 1, nil, true, 1, bufSize}, | |
// Move larger than possible | |
{int64(bufSize), 1, 0, seekErr, false, 0, 0}, | |
// Provide negative offset with CUR_SEEK | |
{int64(-1), 1, 0, seekErr, false, 0, 0}, | |
// Test with whence SEEK_END and with positive offset | |
{1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, | |
// Test with whence SEEK_END and with negative offset | |
{-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, | |
// Test with whence SEEK_END and with large negative offset | |
{-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, | |
} | |
for i, testCase := range testCases { | |
// Perform seek operation | |
n, err := r.Seek(testCase.offset, testCase.whence) | |
// We expect an error | |
if testCase.err == seekErr && err == nil { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) | |
return | |
} | |
// We expect a specific error | |
if testCase.err != seekErr && testCase.err != err { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) | |
return | |
} | |
// If we expect an error go to the next loop | |
if testCase.err != nil { | |
continue | |
} | |
// Check the returned seek pos | |
if n != testCase.pos { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err) | |
return | |
} | |
// Compare only if shouldCmp is activated | |
if testCase.shouldCmp { | |
cmpData(r, testCase.start, testCase.end) | |
} | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests get object ReaderAt interface methods. | |
func testGetObjectReadAtFunctional() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) | |
return | |
} | |
// read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
offset := int64(2048) | |
// read directly | |
buf1 := make([]byte, 512) | |
buf2 := make([]byte, 512) | |
buf3 := make([]byte, 512) | |
buf4 := make([]byte, 512) | |
// Test readAt before stat is called such that objectInfo doesn't change. | |
m, err := r.ReadAt(buf1, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf1) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf1, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) | |
return | |
} | |
m, err = r.ReadAt(buf2, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf2) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf2, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
m, err = r.ReadAt(buf3, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf3) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf3, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
m, err = r.ReadAt(buf4, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf4) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf4, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
buf5 := make([]byte, n) | |
// Read the whole object. | |
m, err = r.ReadAt(buf5, 0) | |
if err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
} | |
if m != len(buf5) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf, buf5) { | |
logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) | |
return | |
} | |
buf6 := make([]byte, n+1) | |
// Read the whole object and beyond. | |
_, err = r.ReadAt(buf6, 0) | |
if err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Reproduces issue https://github.com/minio/minio-go/issues/1137 | |
func testGetObjectReadAtWhenEOFWasReached() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) | |
return | |
} | |
// read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
// read directly | |
buf1 := make([]byte, n) | |
buf2 := make([]byte, 512) | |
m, err := r.Read(buf1) | |
if err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "Read failed", err) | |
return | |
} | |
} | |
if m != len(buf1) { | |
logError(testName, function, args, startTime, "", "Read read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf1, buf) { | |
logError(testName, function, args, startTime, "", "Incorrect count of Read data", err) | |
return | |
} | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) | |
return | |
} | |
m, err = r.ReadAt(buf2, 512) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf2) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf2, buf[512:1024]) { | |
logError(testName, function, args, startTime, "", "Incorrect count of ReadAt data", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test Presigned Post Policy | |
func testPresignedPostPolicy() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PresignedPostPolicy(policy)" | |
args := map[string]interface{}{ | |
"policy": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
// Azure requires the key to not start with a number | |
metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") | |
metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) | |
return | |
} | |
policy := minio.NewPostPolicy() | |
if err := policy.SetBucket(""); err == nil { | |
logError(testName, function, args, startTime, "", "SetBucket did not fail for invalid conditions", err) | |
return | |
} | |
if err := policy.SetKey(""); err == nil { | |
logError(testName, function, args, startTime, "", "SetKey did not fail for invalid conditions", err) | |
return | |
} | |
if err := policy.SetKeyStartsWith(""); err == nil { | |
logError(testName, function, args, startTime, "", "SetKeyStartsWith did not fail for invalid conditions", err) | |
return | |
} | |
if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil { | |
logError(testName, function, args, startTime, "", "SetExpires did not fail for invalid conditions", err) | |
return | |
} | |
if err := policy.SetContentType(""); err == nil { | |
logError(testName, function, args, startTime, "", "SetContentType did not fail for invalid conditions", err) | |
return | |
} | |
if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil { | |
logError(testName, function, args, startTime, "", "SetContentLengthRange did not fail for invalid conditions", err) | |
return | |
} | |
if err := policy.SetUserMetadata("", ""); err == nil { | |
logError(testName, function, args, startTime, "", "SetUserMetadata did not fail for invalid conditions", err) | |
return | |
} | |
policy.SetBucket(bucketName) | |
policy.SetKey(objectName) | |
policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days | |
policy.SetContentType("binary/octet-stream") | |
policy.SetContentLengthRange(10, 1024*1024) | |
policy.SetUserMetadata(metadataKey, metadataValue) | |
args["policy"] = policy.String() | |
presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(policy) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) | |
return | |
} | |
var formBuf bytes.Buffer | |
writer := multipart.NewWriter(&formBuf) | |
for k, v := range formData { | |
writer.WriteField(k, v) | |
} | |
// Get a 33KB file to upload and test if set post policy works | |
var filePath = getMintDataDirFilePath("datafile-33-kB") | |
if filePath == "" { | |
// Make a temp file with 33 KB data. | |
file, err := ioutil.TempFile(os.TempDir(), "PresignedPostPolicyTest") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "TempFile creation failed", err) | |
return | |
} | |
if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil { | |
logError(testName, function, args, startTime, "", "Copy failed", err) | |
return | |
} | |
if err = file.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "File Close failed", err) | |
return | |
} | |
filePath = file.Name() | |
} | |
// add file to post request | |
f, err := os.Open(filePath) | |
defer f.Close() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "File open failed", err) | |
return | |
} | |
w, err := writer.CreateFormFile("file", filePath) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CreateFormFile failed", err) | |
return | |
} | |
_, err = io.Copy(w, f) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Copy failed", err) | |
return | |
} | |
writer.Close() | |
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "DefaultTransport failed", err) | |
return | |
} | |
httpClient := &http.Client{ | |
// Setting a sensible time out of 30secs to wait for response | |
// headers. Request is pro-actively canceled after 30secs | |
// with no response. | |
Timeout: 30 * time.Second, | |
Transport: transport, | |
} | |
req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes())) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Http request failed", err) | |
return | |
} | |
req.Header.Set("Content-Type", writer.FormDataContentType()) | |
// make post request with correct form data | |
res, err := httpClient.Do(req) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Http request failed", err) | |
return | |
} | |
defer res.Body.Close() | |
if res.StatusCode != http.StatusNoContent { | |
logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status)) | |
return | |
} | |
// expected path should be absolute path of the object | |
var scheme string | |
if mustParseBool(os.Getenv(enableHTTPS)) { | |
scheme = "https://" | |
} else { | |
scheme = "http://" | |
} | |
expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName | |
expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName | |
if val, ok := res.Header["Location"]; ok { | |
if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS { | |
logError(testName, function, args, startTime, "", "Location in header response is incorrect", err) | |
return | |
} | |
} else { | |
logError(testName, function, args, startTime, "", "Location not found in header response", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests copy object | |
func testCopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(dst, src)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Make a new bucket in 'us-east-1' (destination bucket). | |
err = c.MakeBucket(bucketName+"-copy", "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) | |
return | |
} | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
// Check the various fields of source object against destination object. | |
objInfo, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
// Copy Source | |
src := minio.NewSourceInfo(bucketName, objectName, nil) | |
args["src"] = src | |
// Set copy conditions. | |
// All invalid conditions first. | |
err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) | |
return | |
} | |
err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) | |
return | |
} | |
err = src.SetMatchETagCond("") | |
if err == nil { | |
logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) | |
return | |
} | |
err = src.SetMatchETagExceptCond("") | |
if err == nil { | |
logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) | |
return | |
} | |
err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) | |
return | |
} | |
err = src.SetMatchETagCond(objInfo.ETag) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) | |
return | |
} | |
dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) | |
args["dst"] = dst | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
// Perform the Copy | |
err = c.CopyObject(dst, src) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed", err) | |
return | |
} | |
// Source object | |
r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
// Destination object | |
readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
// Check the various fields of source object against destination object. | |
objInfo, err = r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
objInfoCopy, err := readerCopy.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if objInfo.Size != objInfoCopy.Size { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err) | |
return | |
} | |
// Close all the get readers before proceeding with CopyObject operations. | |
r.Close() | |
readerCopy.Close() | |
// CopyObject again but with wrong conditions | |
src = minio.NewSourceInfo(bucketName, objectName, nil) | |
err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) | |
return | |
} | |
err = src.SetMatchETagExceptCond(objInfo.ETag) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) | |
return | |
} | |
// Perform the Copy which should fail | |
err = c.CopyObject(dst, src) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) | |
return | |
} | |
// Perform the Copy which should update only metadata. | |
src = minio.NewSourceInfo(bucketName, objectName, nil) | |
dst, err = minio.NewDestinationInfo(bucketName, objectName, nil, map[string]string{ | |
"Copy": "should be same", | |
}) | |
args["dst"] = dst | |
args["src"] = src | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
err = c.CopyObject(dst, src) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err) | |
return | |
} | |
oi, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
stOpts := minio.StatObjectOptions{} | |
stOpts.SetMatchETag(oi.ETag) | |
objInfo, err = c.StatObject(bucketName, objectName, stOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err) | |
return | |
} | |
if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" { | |
logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
if err = cleanupBucket(bucketName+"-copy", c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests SSE-C get object ReaderSeeker interface methods. | |
func testSSECEncryptedGetObjectReadSeekFunctional() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
defer func() { | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
}() | |
// Generate 129MiB of data. | |
bufSize := dataFileMap["datafile-129-MB"] | |
var reader = getDataReader("datafile-129-MB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ | |
ContentType: "binary/octet-stream", | |
ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), | |
}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ | |
ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), | |
}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
defer r.Close() | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat object failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) | |
return | |
} | |
// This following function helps us to compare data from the reader after seek | |
// with the data from the original buffer | |
cmpData := func(r io.Reader, start, end int) { | |
if end-start == 0 { | |
return | |
} | |
buffer := bytes.NewBuffer([]byte{}) | |
if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "CopyN failed", err) | |
return | |
} | |
} | |
if !bytes.Equal(buf[start:end], buffer.Bytes()) { | |
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) | |
return | |
} | |
} | |
testCases := []struct { | |
offset int64 | |
whence int | |
pos int64 | |
err error | |
shouldCmp bool | |
start int | |
end int | |
}{ | |
// Start from offset 0, fetch data and compare | |
{0, 0, 0, nil, true, 0, 0}, | |
// Start from offset 2048, fetch data and compare | |
{2048, 0, 2048, nil, true, 2048, bufSize}, | |
// Start from offset larger than possible | |
{int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, | |
// Move to offset 0 without comparing | |
{0, 0, 0, nil, false, 0, 0}, | |
// Move one step forward and compare | |
{1, 1, 1, nil, true, 1, bufSize}, | |
// Move larger than possible | |
{int64(bufSize), 1, 0, io.EOF, false, 0, 0}, | |
// Provide negative offset with CUR_SEEK | |
{int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, | |
// Test with whence SEEK_END and with positive offset | |
{1024, 2, 0, io.EOF, false, 0, 0}, | |
// Test with whence SEEK_END and with negative offset | |
{-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, | |
// Test with whence SEEK_END and with large negative offset | |
{-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, | |
// Test with invalid whence | |
{0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, | |
} | |
for i, testCase := range testCases { | |
// Perform seek operation | |
n, err := r.Seek(testCase.offset, testCase.whence) | |
if err != nil && testCase.err == nil { | |
// We expected success. | |
logError(testName, function, args, startTime, "", | |
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) | |
return | |
} | |
if err == nil && testCase.err != nil { | |
// We expected failure, but got success. | |
logError(testName, function, args, startTime, "", | |
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) | |
return | |
} | |
if err != nil && testCase.err != nil { | |
if err.Error() != testCase.err.Error() { | |
// We expect a specific error | |
logError(testName, function, args, startTime, "", | |
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) | |
return | |
} | |
} | |
// Check the returned seek pos | |
if n != testCase.pos { | |
logError(testName, function, args, startTime, "", | |
fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) | |
return | |
} | |
// Compare only if shouldCmp is activated | |
if testCase.shouldCmp { | |
cmpData(r, testCase.start, testCase.end) | |
} | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests SSE-S3 get object ReaderSeeker interface methods. | |
func testSSES3EncryptedGetObjectReadSeekFunctional() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
defer func() { | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
}() | |
// Generate 129MiB of data. | |
bufSize := dataFileMap["datafile-129-MB"] | |
var reader = getDataReader("datafile-129-MB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ | |
ContentType: "binary/octet-stream", | |
ServerSideEncryption: encrypt.NewSSE(), | |
}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
defer r.Close() | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat object failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) | |
return | |
} | |
// This following function helps us to compare data from the reader after seek | |
// with the data from the original buffer | |
cmpData := func(r io.Reader, start, end int) { | |
if end-start == 0 { | |
return | |
} | |
buffer := bytes.NewBuffer([]byte{}) | |
if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "CopyN failed", err) | |
return | |
} | |
} | |
if !bytes.Equal(buf[start:end], buffer.Bytes()) { | |
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) | |
return | |
} | |
} | |
testCases := []struct { | |
offset int64 | |
whence int | |
pos int64 | |
err error | |
shouldCmp bool | |
start int | |
end int | |
}{ | |
// Start from offset 0, fetch data and compare | |
{0, 0, 0, nil, true, 0, 0}, | |
// Start from offset 2048, fetch data and compare | |
{2048, 0, 2048, nil, true, 2048, bufSize}, | |
// Start from offset larger than possible | |
{int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, | |
// Move to offset 0 without comparing | |
{0, 0, 0, nil, false, 0, 0}, | |
// Move one step forward and compare | |
{1, 1, 1, nil, true, 1, bufSize}, | |
// Move larger than possible | |
{int64(bufSize), 1, 0, io.EOF, false, 0, 0}, | |
// Provide negative offset with CUR_SEEK | |
{int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, | |
// Test with whence SEEK_END and with positive offset | |
{1024, 2, 0, io.EOF, false, 0, 0}, | |
// Test with whence SEEK_END and with negative offset | |
{-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, | |
// Test with whence SEEK_END and with large negative offset | |
{-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, | |
// Test with invalid whence | |
{0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, | |
} | |
for i, testCase := range testCases { | |
// Perform seek operation | |
n, err := r.Seek(testCase.offset, testCase.whence) | |
if err != nil && testCase.err == nil { | |
// We expected success. | |
logError(testName, function, args, startTime, "", | |
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) | |
return | |
} | |
if err == nil && testCase.err != nil { | |
// We expected failure, but got success. | |
logError(testName, function, args, startTime, "", | |
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) | |
return | |
} | |
if err != nil && testCase.err != nil { | |
if err.Error() != testCase.err.Error() { | |
// We expect a specific error | |
logError(testName, function, args, startTime, "", | |
fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) | |
return | |
} | |
} | |
// Check the returned seek pos | |
if n != testCase.pos { | |
logError(testName, function, args, startTime, "", | |
fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) | |
return | |
} | |
// Compare only if shouldCmp is activated | |
if testCase.shouldCmp { | |
cmpData(r, testCase.start, testCase.end) | |
} | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests SSE-C get object ReaderAt interface methods. | |
func testSSECEncryptedGetObjectReadAtFunctional() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 129MiB of data. | |
bufSize := dataFileMap["datafile-129-MB"] | |
var reader = getDataReader("datafile-129-MB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ | |
ContentType: "binary/octet-stream", | |
ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), | |
}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) | |
return | |
} | |
// read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ | |
ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), | |
}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
defer r.Close() | |
offset := int64(2048) | |
// read directly | |
buf1 := make([]byte, 512) | |
buf2 := make([]byte, 512) | |
buf3 := make([]byte, 512) | |
buf4 := make([]byte, 512) | |
// Test readAt before stat is called such that objectInfo doesn't change. | |
m, err := r.ReadAt(buf1, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf1) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf1, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) | |
return | |
} | |
m, err = r.ReadAt(buf2, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf2) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf2, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
m, err = r.ReadAt(buf3, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf3) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf3, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
m, err = r.ReadAt(buf4, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf4) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf4, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
buf5 := make([]byte, n) | |
// Read the whole object. | |
m, err = r.ReadAt(buf5, 0) | |
if err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
} | |
if m != len(buf5) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf, buf5) { | |
logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) | |
return | |
} | |
buf6 := make([]byte, n+1) | |
// Read the whole object and beyond. | |
_, err = r.ReadAt(buf6, 0) | |
if err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests SSE-S3 get object ReaderAt interface methods. | |
func testSSES3EncryptedGetObjectReadAtFunctional() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 129MiB of data. | |
bufSize := dataFileMap["datafile-129-MB"] | |
var reader = getDataReader("datafile-129-MB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ | |
ContentType: "binary/octet-stream", | |
ServerSideEncryption: encrypt.NewSSE(), | |
}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) | |
return | |
} | |
// read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
defer r.Close() | |
offset := int64(2048) | |
// read directly | |
buf1 := make([]byte, 512) | |
buf2 := make([]byte, 512) | |
buf3 := make([]byte, 512) | |
buf4 := make([]byte, 512) | |
// Test readAt before stat is called such that objectInfo doesn't change. | |
m, err := r.ReadAt(buf1, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf1) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf1, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) | |
return | |
} | |
m, err = r.ReadAt(buf2, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf2) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf2, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
m, err = r.ReadAt(buf3, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf3) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf3, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
m, err = r.ReadAt(buf4, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf4) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf4, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
buf5 := make([]byte, n) | |
// Read the whole object. | |
m, err = r.ReadAt(buf5, 0) | |
if err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
} | |
if m != len(buf5) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf, buf5) { | |
logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) | |
return | |
} | |
buf6 := make([]byte, n+1) | |
// Read the whole object and beyond. | |
_, err = r.ReadAt(buf6, 0) | |
if err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// testSSECEncryptionPutGet tests encryption with customer provided encryption keys | |
func testSSECEncryptionPutGet() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutEncryptedObject(bucketName, objectName, reader, sse)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"sse": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
testCases := []struct { | |
buf []byte | |
}{ | |
{buf: bytes.Repeat([]byte("F"), 1)}, | |
{buf: bytes.Repeat([]byte("F"), 15)}, | |
{buf: bytes.Repeat([]byte("F"), 16)}, | |
{buf: bytes.Repeat([]byte("F"), 17)}, | |
{buf: bytes.Repeat([]byte("F"), 31)}, | |
{buf: bytes.Repeat([]byte("F"), 32)}, | |
{buf: bytes.Repeat([]byte("F"), 33)}, | |
{buf: bytes.Repeat([]byte("F"), 1024)}, | |
{buf: bytes.Repeat([]byte("F"), 1024*2)}, | |
{buf: bytes.Repeat([]byte("F"), 1024*1024)}, | |
} | |
const password = "correct horse battery staple" // https://xkcd.com/936/ | |
for i, testCase := range testCases { | |
// Generate a random object name | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Secured object | |
sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) | |
args["sse"] = sse | |
// Put encrypted data | |
_, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) | |
return | |
} | |
defer r.Close() | |
// Compare the sent object with the received one | |
recvBuffer := bytes.NewBuffer([]byte{}) | |
if _, err = io.Copy(recvBuffer, r); err != nil { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) | |
return | |
} | |
if recvBuffer.Len() != len(testCase.buf) { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) | |
return | |
} | |
if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// TestEncryptionFPut tests encryption with customer specified encryption keys | |
func testSSECEncryptionFPut() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"filePath": "", | |
"contentType": "", | |
"sse": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Object custom metadata | |
customContentType := "custom/contenttype" | |
args["metadata"] = customContentType | |
testCases := []struct { | |
buf []byte | |
}{ | |
{buf: bytes.Repeat([]byte("F"), 0)}, | |
{buf: bytes.Repeat([]byte("F"), 1)}, | |
{buf: bytes.Repeat([]byte("F"), 15)}, | |
{buf: bytes.Repeat([]byte("F"), 16)}, | |
{buf: bytes.Repeat([]byte("F"), 17)}, | |
{buf: bytes.Repeat([]byte("F"), 31)}, | |
{buf: bytes.Repeat([]byte("F"), 32)}, | |
{buf: bytes.Repeat([]byte("F"), 33)}, | |
{buf: bytes.Repeat([]byte("F"), 1024)}, | |
{buf: bytes.Repeat([]byte("F"), 1024*2)}, | |
{buf: bytes.Repeat([]byte("F"), 1024*1024)}, | |
} | |
const password = "correct horse battery staple" // https://xkcd.com/936/ | |
for i, testCase := range testCases { | |
// Generate a random object name | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Secured object | |
sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) | |
args["sse"] = sse | |
// Generate a random file name. | |
fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
file, err := os.Create(fileName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "file create failed", err) | |
return | |
} | |
_, err = file.Write(testCase.buf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "file write failed", err) | |
return | |
} | |
file.Close() | |
// Put encrypted data | |
if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { | |
logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) | |
return | |
} | |
defer r.Close() | |
// Compare the sent object with the received one | |
recvBuffer := bytes.NewBuffer([]byte{}) | |
if _, err = io.Copy(recvBuffer, r); err != nil { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) | |
return | |
} | |
if recvBuffer.Len() != len(testCase.buf) { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) | |
return | |
} | |
if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) | |
return | |
} | |
os.Remove(fileName) | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// testSSES3EncryptionPutGet tests SSE-S3 encryption | |
func testSSES3EncryptionPutGet() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutEncryptedObject(bucketName, objectName, reader, sse)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"sse": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
testCases := []struct { | |
buf []byte | |
}{ | |
{buf: bytes.Repeat([]byte("F"), 1)}, | |
{buf: bytes.Repeat([]byte("F"), 15)}, | |
{buf: bytes.Repeat([]byte("F"), 16)}, | |
{buf: bytes.Repeat([]byte("F"), 17)}, | |
{buf: bytes.Repeat([]byte("F"), 31)}, | |
{buf: bytes.Repeat([]byte("F"), 32)}, | |
{buf: bytes.Repeat([]byte("F"), 33)}, | |
{buf: bytes.Repeat([]byte("F"), 1024)}, | |
{buf: bytes.Repeat([]byte("F"), 1024*2)}, | |
{buf: bytes.Repeat([]byte("F"), 1024*1024)}, | |
} | |
for i, testCase := range testCases { | |
// Generate a random object name | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Secured object | |
sse := encrypt.NewSSE() | |
args["sse"] = sse | |
// Put encrypted data | |
_, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) | |
return | |
} | |
// Read the data back without any encryption headers | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) | |
return | |
} | |
defer r.Close() | |
// Compare the sent object with the received one | |
recvBuffer := bytes.NewBuffer([]byte{}) | |
if _, err = io.Copy(recvBuffer, r); err != nil { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) | |
return | |
} | |
if recvBuffer.Len() != len(testCase.buf) { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) | |
return | |
} | |
if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// TestSSES3EncryptionFPut tests server side encryption | |
func testSSES3EncryptionFPut() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"filePath": "", | |
"contentType": "", | |
"sse": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Object custom metadata | |
customContentType := "custom/contenttype" | |
args["metadata"] = customContentType | |
testCases := []struct { | |
buf []byte | |
}{ | |
{buf: bytes.Repeat([]byte("F"), 0)}, | |
{buf: bytes.Repeat([]byte("F"), 1)}, | |
{buf: bytes.Repeat([]byte("F"), 15)}, | |
{buf: bytes.Repeat([]byte("F"), 16)}, | |
{buf: bytes.Repeat([]byte("F"), 17)}, | |
{buf: bytes.Repeat([]byte("F"), 31)}, | |
{buf: bytes.Repeat([]byte("F"), 32)}, | |
{buf: bytes.Repeat([]byte("F"), 33)}, | |
{buf: bytes.Repeat([]byte("F"), 1024)}, | |
{buf: bytes.Repeat([]byte("F"), 1024*2)}, | |
{buf: bytes.Repeat([]byte("F"), 1024*1024)}, | |
} | |
for i, testCase := range testCases { | |
// Generate a random object name | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Secured object | |
sse := encrypt.NewSSE() | |
args["sse"] = sse | |
// Generate a random file name. | |
fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
file, err := os.Create(fileName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "file create failed", err) | |
return | |
} | |
_, err = file.Write(testCase.buf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "file write failed", err) | |
return | |
} | |
file.Close() | |
// Put encrypted data | |
if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { | |
logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) | |
return | |
} | |
defer r.Close() | |
// Compare the sent object with the received one | |
recvBuffer := bytes.NewBuffer([]byte{}) | |
if _, err = io.Copy(recvBuffer, r); err != nil { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) | |
return | |
} | |
if recvBuffer.Len() != len(testCase.buf) { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) | |
return | |
} | |
if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { | |
logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) | |
return | |
} | |
os.Remove(fileName) | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testBucketNotification() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "SetBucketNotification(bucketName)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
} | |
if os.Getenv("NOTIFY_BUCKET") == "" || | |
os.Getenv("NOTIFY_SERVICE") == "" || | |
os.Getenv("NOTIFY_REGION") == "" || | |
os.Getenv("NOTIFY_ACCOUNTID") == "" || | |
os.Getenv("NOTIFY_RESOURCE") == "" { | |
ignoredLog(testName, function, args, startTime, "Skipped notification test as it is not configured").Info() | |
return | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable to debug | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
bucketName := os.Getenv("NOTIFY_BUCKET") | |
args["bucketName"] = bucketName | |
topicArn := minio.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) | |
queueArn := minio.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") | |
topicConfig := minio.NewNotificationConfig(topicArn) | |
topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) | |
topicConfig.AddFilterSuffix("jpg") | |
queueConfig := minio.NewNotificationConfig(queueArn) | |
queueConfig.AddEvents(minio.ObjectCreatedAll) | |
queueConfig.AddFilterPrefix("photos/") | |
bNotification := minio.BucketNotification{} | |
bNotification.AddTopic(topicConfig) | |
// Add the same topicConfig again, should have no effect | |
// because it is duplicated | |
bNotification.AddTopic(topicConfig) | |
if len(bNotification.TopicConfigs) != 1 { | |
logError(testName, function, args, startTime, "", "Duplicate entry added", err) | |
return | |
} | |
// Add and remove a queue config | |
bNotification.AddQueue(queueConfig) | |
bNotification.RemoveQueueByArn(queueArn) | |
err = c.SetBucketNotification(bucketName, bNotification) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetBucketNotification failed", err) | |
return | |
} | |
bNotification, err = c.GetBucketNotification(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetBucketNotification failed", err) | |
return | |
} | |
if len(bNotification.TopicConfigs) != 1 { | |
logError(testName, function, args, startTime, "", "Topic config is empty", err) | |
return | |
} | |
if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { | |
logError(testName, function, args, startTime, "", "Couldn't get the suffix", err) | |
return | |
} | |
err = c.RemoveAllBucketNotification(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests comprehensive list of all methods. | |
func testFunctional() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "testFunctional()" | |
functionAll := "" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable to debug | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
// Make a new bucket. | |
function = "MakeBucket(bucketName, region)" | |
functionAll = "MakeBucket(bucketName, region)" | |
args["bucketName"] = bucketName | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate a random file name. | |
fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
file, err := os.Create(fileName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "File creation failed", err) | |
return | |
} | |
for i := 0; i < 3; i++ { | |
buf := make([]byte, rand.Intn(1<<19)) | |
_, err = file.Write(buf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "File write failed", err) | |
return | |
} | |
} | |
file.Close() | |
// Verify if bucket exits and you have access. | |
var exists bool | |
function = "BucketExists(bucketName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
} | |
exists, err = c.BucketExists(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "BucketExists failed", err) | |
return | |
} | |
if !exists { | |
logError(testName, function, args, startTime, "", "Could not find the bucket", err) | |
return | |
} | |
// Verify if bucket exits and you have access with context. | |
function = "BucketExistsWithContext(ctx, bucketName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
} | |
exists, err = c.BucketExistsWithContext(context.Background(), bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "BucketExistsWithContext failed", err) | |
return | |
} | |
if !exists { | |
logError(testName, function, args, startTime, "", "Could not find the bucket", err) | |
return | |
} | |
// Asserting the default bucket policy. | |
function = "GetBucketPolicy(bucketName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
} | |
nilPolicy, err := c.GetBucketPolicy(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) | |
return | |
} | |
if nilPolicy != "" { | |
logError(testName, function, args, startTime, "", "policy should be set to nil", err) | |
return | |
} | |
// Set the bucket policy to 'public readonly'. | |
function = "SetBucketPolicy(bucketName, readOnlyPolicy)" | |
functionAll += ", " + function | |
readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"bucketPolicy": readOnlyPolicy, | |
} | |
err = c.SetBucketPolicy(bucketName, readOnlyPolicy) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) | |
return | |
} | |
// should return policy `readonly`. | |
function = "GetBucketPolicy(bucketName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
} | |
_, err = c.GetBucketPolicy(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) | |
return | |
} | |
// Make the bucket 'public writeonly'. | |
function = "SetBucketPolicy(bucketName, writeOnlyPolicy)" | |
functionAll += ", " + function | |
writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"bucketPolicy": writeOnlyPolicy, | |
} | |
err = c.SetBucketPolicy(bucketName, writeOnlyPolicy) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) | |
return | |
} | |
// should return policy `writeonly`. | |
function = "GetBucketPolicy(bucketName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
} | |
_, err = c.GetBucketPolicy(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) | |
return | |
} | |
// Make the bucket 'public read/write'. | |
function = "SetBucketPolicy(bucketName, readWritePolicy)" | |
functionAll += ", " + function | |
readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"bucketPolicy": readWritePolicy, | |
} | |
err = c.SetBucketPolicy(bucketName, readWritePolicy) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) | |
return | |
} | |
// should return policy `readwrite`. | |
function = "GetBucketPolicy(bucketName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
} | |
_, err = c.GetBucketPolicy(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) | |
return | |
} | |
// List all buckets. | |
function = "ListBuckets()" | |
functionAll += ", " + function | |
args = nil | |
buckets, err := c.ListBuckets() | |
if len(buckets) == 0 { | |
logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) | |
return | |
} | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ListBuckets failed", err) | |
return | |
} | |
// List all buckets with context. | |
function = "ListBucketsWithContext()" | |
functionAll += ", " + function | |
args = nil | |
buckets, err = c.ListBucketsWithContext(context.Background()) | |
if len(buckets) == 0 { | |
logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) | |
return | |
} | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ListBucketsWithContext failed", err) | |
return | |
} | |
// Verify if previously created bucket is listed in list buckets. | |
bucketFound := false | |
for _, bucket := range buckets { | |
if bucket.Name == bucketName { | |
bucketFound = true | |
} | |
} | |
// If bucket not found error out. | |
if !bucketFound { | |
logError(testName, function, args, startTime, "", "Bucket: "+bucketName+" not found", err) | |
return | |
} | |
objectName := bucketName + "unique" | |
// Generate data | |
buf := bytes.Repeat([]byte("f"), 1<<19) | |
function = "PutObject(bucketName, objectName, reader, contentType)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"contentType": "", | |
} | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) | |
return | |
} | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName + "-nolength", | |
"contentType": "binary/octet-stream", | |
} | |
n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) | |
return | |
} | |
// Instantiate a done channel to close all listing. | |
doneCh := make(chan struct{}) | |
defer close(doneCh) | |
objFound := false | |
isRecursive := true // Recursive is true. | |
function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"isRecursive": isRecursive, | |
} | |
for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { | |
if obj.Key == objectName { | |
objFound = true | |
break | |
} | |
} | |
if !objFound { | |
logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) | |
return | |
} | |
objFound = false | |
isRecursive = true // Recursive is true. | |
function = "ListObjectsV2(bucketName, objectName, isRecursive, doneCh)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"isRecursive": isRecursive, | |
} | |
for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { | |
if obj.Key == objectName { | |
objFound = true | |
break | |
} | |
} | |
if !objFound { | |
logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) | |
return | |
} | |
incompObjNotFound := true | |
function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"isRecursive": isRecursive, | |
} | |
for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { | |
if objIncompl.Key != "" { | |
incompObjNotFound = false | |
break | |
} | |
} | |
if !incompObjNotFound { | |
logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) | |
return | |
} | |
function = "GetObject(bucketName, objectName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
} | |
newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
newReadBytes, err := ioutil.ReadAll(newReader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
if !bytes.Equal(newReadBytes, buf) { | |
logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err) | |
return | |
} | |
newReader.Close() | |
function = "FGetObject(bucketName, objectName, fileName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"fileName": fileName + "-f", | |
} | |
err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FGetObject failed", err) | |
return | |
} | |
function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": "", | |
"expires": 3600 * time.Second, | |
} | |
if _, err = c.PresignedHeadObject(bucketName, "", 3600*time.Second, nil); err == nil { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject success", err) | |
return | |
} | |
// Generate presigned HEAD object url. | |
function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"expires": 3600 * time.Second, | |
} | |
presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) | |
return | |
} | |
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "DefaultTransport failed", err) | |
return | |
} | |
httpClient := &http.Client{ | |
// Setting a sensible time out of 30secs to wait for response | |
// headers. Request is pro-actively canceled after 30secs | |
// with no response. | |
Timeout: 30 * time.Second, | |
Transport: transport, | |
} | |
req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject request was incorrect", err) | |
return | |
} | |
// Verify if presigned url works. | |
resp, err := httpClient.Do(req) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) | |
return | |
} | |
if resp.StatusCode != http.StatusOK { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err) | |
return | |
} | |
if resp.Header.Get("ETag") == "" { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) | |
return | |
} | |
resp.Body.Close() | |
function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": "", | |
"expires": 3600 * time.Second, | |
} | |
_, err = c.PresignedGetObject(bucketName, "", 3600*time.Second, nil) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject success", err) | |
return | |
} | |
// Generate presigned GET object url. | |
function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"expires": 3600 * time.Second, | |
} | |
presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) | |
return | |
} | |
// Verify if presigned url works. | |
req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) | |
return | |
} | |
resp, err = httpClient.Do(req) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) | |
return | |
} | |
if resp.StatusCode != http.StatusOK { | |
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) | |
return | |
} | |
newPresignedBytes, err := ioutil.ReadAll(resp.Body) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) | |
return | |
} | |
resp.Body.Close() | |
if !bytes.Equal(newPresignedBytes, buf) { | |
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) | |
return | |
} | |
// Set request parameters. | |
reqParams := make(url.Values) | |
reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"expires": 3600 * time.Second, | |
"reqParams": reqParams, | |
} | |
presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) | |
return | |
} | |
// Verify if presigned url works. | |
req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) | |
return | |
} | |
resp, err = httpClient.Do(req) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) | |
return | |
} | |
if resp.StatusCode != http.StatusOK { | |
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) | |
return | |
} | |
newPresignedBytes, err = ioutil.ReadAll(resp.Body) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) | |
return | |
} | |
if !bytes.Equal(newPresignedBytes, buf) { | |
logError(testName, function, args, startTime, "", "Bytes mismatch for presigned GET URL", err) | |
return | |
} | |
if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { | |
logError(testName, function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err) | |
return | |
} | |
function = "PresignedPutObject(bucketName, objectName, expires)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": "", | |
"expires": 3600 * time.Second, | |
} | |
_, err = c.PresignedPutObject(bucketName, "", 3600*time.Second) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "PresignedPutObject success", err) | |
return | |
} | |
function = "PresignedPutObject(bucketName, objectName, expires)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName + "-presigned", | |
"expires": 3600 * time.Second, | |
} | |
presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) | |
return | |
} | |
buf = bytes.Repeat([]byte("g"), 1<<19) | |
req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err) | |
return | |
} | |
resp, err = httpClient.Do(req) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) | |
return | |
} | |
newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err) | |
return | |
} | |
newReadBytes, err = ioutil.ReadAll(newReader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err) | |
return | |
} | |
if !bytes.Equal(newReadBytes, buf) { | |
logError(testName, function, args, startTime, "", "Bytes mismatch", err) | |
return | |
} | |
function = "RemoveObject(bucketName, objectName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
} | |
err = c.RemoveObject(bucketName, objectName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "RemoveObject failed", err) | |
return | |
} | |
args["objectName"] = objectName + "-f" | |
err = c.RemoveObject(bucketName, objectName+"-f") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "RemoveObject failed", err) | |
return | |
} | |
args["objectName"] = objectName + "-nolength" | |
err = c.RemoveObject(bucketName, objectName+"-nolength") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "RemoveObject failed", err) | |
return | |
} | |
args["objectName"] = objectName + "-presigned" | |
err = c.RemoveObject(bucketName, objectName+"-presigned") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "RemoveObject failed", err) | |
return | |
} | |
function = "RemoveBucket(bucketName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
} | |
err = c.RemoveBucket(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "RemoveBucket failed", err) | |
return | |
} | |
err = c.RemoveBucket(bucketName) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err) | |
return | |
} | |
if err.Error() != "The specified bucket does not exist" { | |
logError(testName, function, args, startTime, "", "RemoveBucket failed", err) | |
return | |
} | |
os.Remove(fileName) | |
os.Remove(fileName + "-f") | |
successLogger(testName, functionAll, args, startTime).Info() | |
} | |
// Test for validating GetObject Reader* methods functioning when the | |
// object is modified in the object store. | |
func testGetObjectModified() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object. | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Make a new bucket. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
defer c.RemoveBucket(bucketName) | |
// Upload an object. | |
objectName := "myobject" | |
args["objectName"] = objectName | |
content := "helloworld" | |
_, err = c.PutObject(bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) | |
return | |
} | |
defer c.RemoveObject(bucketName, objectName) | |
reader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err) | |
return | |
} | |
defer reader.Close() | |
// Read a few bytes of the object. | |
b := make([]byte, 5) | |
n, err := reader.ReadAt(b, 0) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err) | |
return | |
} | |
// Upload different contents to the same object while object is being read. | |
newContent := "goodbyeworld" | |
_, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) | |
return | |
} | |
// Confirm that a Stat() call in between doesn't change the Object's cached etag. | |
_, err = reader.Stat() | |
expectedError := "At least one of the pre-conditions you specified did not hold" | |
if err.Error() != expectedError { | |
logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err) | |
return | |
} | |
// Read again only to find object contents have been modified since last read. | |
_, err = reader.ReadAt(b, int64(n)) | |
if err.Error() != expectedError { | |
logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test validates putObject to upload a file seeked at a given offset. | |
func testPutObjectUploadSeekedObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, fileToUpload, contentType)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"fileToUpload": "", | |
"contentType": "binary/octet-stream", | |
} | |
// Instantiate new minio client object. | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Make a new bucket. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
defer c.RemoveBucket(bucketName) | |
var tempfile *os.File | |
if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" { | |
tempfile, err = os.Open(fileName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "File open failed", err) | |
return | |
} | |
args["fileToUpload"] = fileName | |
} else { | |
tempfile, err = ioutil.TempFile("", "minio-go-upload-test-") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "TempFile create failed", err) | |
return | |
} | |
args["fileToUpload"] = tempfile.Name() | |
// Generate 100kB data | |
if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil { | |
logError(testName, function, args, startTime, "", "File copy failed", err) | |
return | |
} | |
defer os.Remove(tempfile.Name()) | |
// Seek back to the beginning of the file. | |
tempfile.Seek(0, 0) | |
} | |
var length = 100 * humanize.KiByte | |
objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) | |
args["objectName"] = objectName | |
offset := length / 2 | |
if _, err = tempfile.Seek(int64(offset), 0); err != nil { | |
logError(testName, function, args, startTime, "", "TempFile seek failed", err) | |
return | |
} | |
n, err := c.PutObject(bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(length-offset) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid length returned, expected %d got %d", int64(length-offset), n), err) | |
return | |
} | |
tempfile.Close() | |
obj, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
defer obj.Close() | |
n, err = obj.Seek(int64(offset), 0) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Seek failed", err) | |
return | |
} | |
if n != int64(offset) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err) | |
return | |
} | |
n, err = c.PutObject(bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(length-offset) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests bucket re-create errors. | |
func testMakeBucketErrorV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "MakeBucket(bucketName, region)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"region": "eu-west-1", | |
} | |
if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { | |
ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() | |
return | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
region := "eu-west-1" | |
args["bucketName"] = bucketName | |
args["region"] = region | |
// Make a new bucket in 'eu-west-1'. | |
if err = c.MakeBucket(bucketName, region); err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
if err = c.MakeBucket(bucketName, region); err == nil { | |
logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) | |
return | |
} | |
// Verify valid error response from server. | |
if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && | |
minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { | |
logError(testName, function, args, startTime, "", "Invalid error returned by server", err) | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test get object reader to not throw error on being closed twice. | |
func testGetObjectClosedTwiceV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "MakeBucket(bucketName, region)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"region": "eu-west-1", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) | |
return | |
} | |
if err := r.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if err := r.Close(); err == nil { | |
logError(testName, function, args, startTime, "", "Object is already closed, should return error", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests FPutObject hidden contentType setting | |
func testFPutObjectV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "FPutObject(bucketName, objectName, fileName, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"fileName": "", | |
"opts": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Make a temp file with 11*1024*1024 bytes of data. | |
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "TempFile creation failed", err) | |
return | |
} | |
r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) | |
n, err := io.CopyN(file, r, 11*1024*1024) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Copy failed", err) | |
return | |
} | |
if n != int64(11*1024*1024) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) | |
return | |
} | |
// Close the file pro-actively for windows. | |
err = file.Close() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "File close failed", err) | |
return | |
} | |
// Set base object name | |
objectName := bucketName + "FPutObject" | |
args["objectName"] = objectName | |
args["fileName"] = file.Name() | |
// Perform standard FPutObject with contentType provided (Expecting application/octet-stream) | |
n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FPutObject failed", err) | |
return | |
} | |
if n != int64(11*1024*1024) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) | |
return | |
} | |
// Perform FPutObject with no contentType provided (Expecting application/octet-stream) | |
args["objectName"] = objectName + "-Octet" | |
args["contentType"] = "" | |
n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FPutObject failed", err) | |
return | |
} | |
if n != int64(11*1024*1024) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) | |
return | |
} | |
// Add extension to temp file name | |
fileName := file.Name() | |
err = os.Rename(fileName, fileName+".gtar") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Rename failed", err) | |
return | |
} | |
// Perform FPutObject with no contentType provided (Expecting application/x-gtar) | |
args["objectName"] = objectName + "-Octet" | |
args["contentType"] = "" | |
args["fileName"] = fileName + ".gtar" | |
n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FPutObject failed", err) | |
return | |
} | |
if n != int64(11*1024*1024) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) | |
return | |
} | |
// Check headers | |
rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
if rStandard.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err) | |
return | |
} | |
rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
if rOctet.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err) | |
return | |
} | |
rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) | |
return | |
} | |
rGTar, err = c.StatObjectWithContext(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { | |
logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
os.Remove(fileName + ".gtar") | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests various bucket supported formats. | |
func testMakeBucketRegionsV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "MakeBucket(bucketName, region)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"region": "eu-west-1", | |
} | |
if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { | |
ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() | |
return | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket in 'eu-central-1'. | |
if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
// Make a new bucket with '.' in its name, in 'us-west-2'. This | |
// request is internally staged into a path style instead of | |
// virtual host style. | |
if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { | |
args["bucketName"] = bucketName + ".withperiod" | |
args["region"] = "us-west-2" | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName+".withperiod", c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests get object ReaderSeeker interface methods. | |
func testGetObjectReadSeekFunctionalV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data. | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
defer r.Close() | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) | |
return | |
} | |
offset := int64(2048) | |
n, err = r.Seek(offset, 0) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Seek failed", err) | |
return | |
} | |
if n != offset { | |
logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) | |
return | |
} | |
n, err = r.Seek(0, 1) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Seek failed", err) | |
return | |
} | |
if n != offset { | |
logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) | |
return | |
} | |
_, err = r.Seek(offset, 2) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err) | |
return | |
} | |
n, err = r.Seek(-offset, 2) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Seek failed", err) | |
return | |
} | |
if n != st.Size-offset { | |
logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err) | |
return | |
} | |
var buffer1 bytes.Buffer | |
if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "Copy failed", err) | |
return | |
} | |
} | |
if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { | |
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) | |
return | |
} | |
// Seek again and read again. | |
n, err = r.Seek(offset-1, 0) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Seek failed", err) | |
return | |
} | |
if n != (offset - 1) { | |
logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err) | |
return | |
} | |
var buffer2 bytes.Buffer | |
if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "Copy failed", err) | |
return | |
} | |
} | |
// Verify now lesser bytes. | |
if !bytes.Equal(buf[2047:], buffer2.Bytes()) { | |
logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests get object ReaderAt interface methods. | |
func testGetObjectReadAtFunctionalV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObject(bucketName, objectName)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
buf, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
// Save the data | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) | |
return | |
} | |
// Read the data back | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
defer r.Close() | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) | |
return | |
} | |
offset := int64(2048) | |
// Read directly | |
buf2 := make([]byte, 512) | |
buf3 := make([]byte, 512) | |
buf4 := make([]byte, 512) | |
m, err := r.ReadAt(buf2, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf2) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf2, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
m, err = r.ReadAt(buf3, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf3) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf3, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
offset += 512 | |
m, err = r.ReadAt(buf4, offset) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
if m != len(buf4) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf4, buf[offset:offset+512]) { | |
logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) | |
return | |
} | |
buf5 := make([]byte, n) | |
// Read the whole object. | |
m, err = r.ReadAt(buf5, 0) | |
if err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
} | |
if m != len(buf5) { | |
logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err) | |
return | |
} | |
if !bytes.Equal(buf, buf5) { | |
logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) | |
return | |
} | |
buf6 := make([]byte, n+1) | |
// Read the whole object and beyond. | |
_, err = r.ReadAt(buf6, 0) | |
if err != nil { | |
if err != io.EOF { | |
logError(testName, function, args, startTime, "", "ReadAt failed", err) | |
return | |
} | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Tests copy object | |
func testCopyObjectV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Make a new bucket in 'us-east-1' (destination bucket). | |
err = c.MakeBucket(bucketName+"-copy", "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate 33K of data. | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) | |
return | |
} | |
r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
// Check the various fields of source object against destination object. | |
objInfo, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
r.Close() | |
// Copy Source | |
src := minio.NewSourceInfo(bucketName, objectName, nil) | |
args["source"] = src | |
// Set copy conditions. | |
// All invalid conditions first. | |
err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) | |
return | |
} | |
err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) | |
return | |
} | |
err = src.SetMatchETagCond("") | |
if err == nil { | |
logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) | |
return | |
} | |
err = src.SetMatchETagExceptCond("") | |
if err == nil { | |
logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) | |
return | |
} | |
err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) | |
return | |
} | |
err = src.SetMatchETagCond(objInfo.ETag) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) | |
return | |
} | |
dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) | |
args["destination"] = dst | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
// Perform the Copy | |
err = c.CopyObject(dst, src) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed", err) | |
return | |
} | |
// Source object | |
r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
// Destination object | |
readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
// Check the various fields of source object against destination object. | |
objInfo, err = r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
objInfoCopy, err := readerCopy.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
if objInfo.Size != objInfoCopy.Size { | |
logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err) | |
return | |
} | |
// Close all the readers. | |
r.Close() | |
readerCopy.Close() | |
// CopyObject again but with wrong conditions | |
src = minio.NewSourceInfo(bucketName, objectName, nil) | |
err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) | |
return | |
} | |
err = src.SetMatchETagExceptCond(objInfo.ETag) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) | |
return | |
} | |
// Perform the Copy which should fail | |
err = c.CopyObject(dst, src) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
if err = cleanupBucket(bucketName+"-copy", c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testComposeObjectErrorCasesWrapper(c *minio.Client) { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "ComposeObject(destination, sourceList)" | |
args := map[string]interface{}{} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err := c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Test that more than 10K source objects cannot be | |
// concatenated. | |
srcArr := [10001]minio.SourceInfo{} | |
srcSlice := srcArr[:] | |
dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
args["destination"] = dst | |
// Just explain about srcArr in args["sourceList"] | |
// to stop having 10,001 null headers logged | |
args["sourceList"] = "source array of 10,001 elements" | |
if err := c.ComposeObject(dst, srcSlice); err == nil { | |
logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) | |
return | |
} else if err.Error() != "There must be as least one and up to 10000 source objects." { | |
logError(testName, function, args, startTime, "", "Got unexpected error", err) | |
return | |
} | |
// Create a source with invalid offset spec and check that | |
// error is returned: | |
// 1. Create the source object. | |
const badSrcSize = 5 * 1024 * 1024 | |
buf := bytes.Repeat([]byte("1"), badSrcSize) | |
_, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
// 2. Set invalid range spec on the object (going beyond | |
// object size) | |
badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) | |
err = badSrc.SetRange(1, badSrcSize) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Setting NewSourceInfo failed", err) | |
return | |
} | |
// 3. ComposeObject call should fail. | |
if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { | |
logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) | |
return | |
} else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { | |
logError(testName, function, args, startTime, "", "Got invalid error", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test expected error cases | |
func testComposeObjectErrorCasesV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "ComposeObject(destination, sourceList)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
testComposeObjectErrorCasesWrapper(c) | |
} | |
func testComposeMultipleSources(c *minio.Client) { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "ComposeObject(destination, sourceList)" | |
args := map[string]interface{}{ | |
"destination": "", | |
"sourceList": "", | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err := c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Upload a small source object | |
const srcSize = 1024 * 1024 * 5 | |
buf := bytes.Repeat([]byte("1"), srcSize) | |
_, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
// We will append 10 copies of the object. | |
srcs := []minio.SourceInfo{} | |
for i := 0; i < 10; i++ { | |
srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) | |
} | |
// make the last part very small | |
err = srcs[9].SetRange(0, 0) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetRange failed", err) | |
return | |
} | |
args["sourceList"] = srcs | |
dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) | |
args["destination"] = dst | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
err = c.ComposeObject(dst, srcs) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ComposeObject failed", err) | |
return | |
} | |
objProps, err := c.StatObject(bucketName, "dstObject", minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject failed", err) | |
return | |
} | |
if objProps.Size != 9*srcSize+1 { | |
logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test concatenating multiple objects objects | |
func testCompose10KSourcesV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "ComposeObject(destination, sourceList)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
testComposeMultipleSources(c) | |
} | |
func testEncryptedEmptyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, reader, objectSize, opts)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object")) | |
// 1. create an sse-c encrypted object to copy by uploading | |
const srcSize = 0 | |
var buf []byte // Empty buffer | |
args["objectName"] = "object" | |
_, err = c.PutObject(bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
return | |
} | |
// 2. Test CopyObject for an empty object | |
dstInfo, err := minio.NewDestinationInfo(bucketName, "new-object", sse, nil) | |
if err != nil { | |
args["objectName"] = "new-object" | |
function = "NewDestinationInfo(bucketName, objectName, sse, userMetadata)" | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
srcInfo := minio.NewSourceInfo(bucketName, "object", sse) | |
if err = c.CopyObject(dstInfo, srcInfo); err != nil { | |
function = "CopyObject(dstInfo, srcInfo)" | |
logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) | |
return | |
} | |
// 3. Test Key rotation | |
newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object")) | |
dstInfo, err = minio.NewDestinationInfo(bucketName, "new-object", newSSE, nil) | |
if err != nil { | |
args["objectName"] = "new-object" | |
function = "NewDestinationInfo(bucketName, objectName, encryptSSEC, userMetadata)" | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
srcInfo = minio.NewSourceInfo(bucketName, "new-object", sse) | |
if err = c.CopyObject(dstInfo, srcInfo); err != nil { | |
function = "CopyObject(dstInfo, srcInfo)" | |
logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) | |
return | |
} | |
// 4. Download the object. | |
reader, err := c.GetObject(bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
defer reader.Close() | |
decBytes, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err) | |
return | |
} | |
if !bytes.Equal(decBytes, buf) { | |
logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err) | |
return | |
} | |
// Delete all objects and buckets | |
delete(args, "objectName") | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, sseDst encrypt.ServerSide) { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
var srcEncryption, dstEncryption encrypt.ServerSide | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err := c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// 1. create an sse-c encrypted object to copy by uploading | |
const srcSize = 1024 * 1024 | |
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB | |
_, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ | |
ServerSideEncryption: sseSrc, | |
}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
return | |
} | |
if sseSrc != nil && sseSrc.Type() != encrypt.S3 { | |
srcEncryption = sseSrc | |
} | |
// 2. copy object and change encryption key | |
src := minio.NewSourceInfo(bucketName, "srcObject", srcEncryption) | |
args["source"] = src | |
dst, err := minio.NewDestinationInfo(bucketName, "dstObject", sseDst, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
args["destination"] = dst | |
err = c.CopyObject(dst, src) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed", err) | |
return | |
} | |
if sseDst != nil && sseDst.Type() != encrypt.S3 { | |
dstEncryption = sseDst | |
} | |
// 3. get copied object and check if content is equal | |
coreClient := minio.Core{c} | |
reader, _, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
decBytes, err := ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
if !bytes.Equal(decBytes, buf) { | |
logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) | |
return | |
} | |
reader.Close() | |
// Test key rotation for source object in-place. | |
var newSSE encrypt.ServerSide | |
if sseSrc != nil && sseSrc.Type() == encrypt.SSEC { | |
newSSE = encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key | |
} | |
if sseSrc != nil && sseSrc.Type() == encrypt.S3 { | |
newSSE = encrypt.NewSSE() | |
} | |
if newSSE != nil { | |
dst, err = minio.NewDestinationInfo(bucketName, "srcObject", newSSE, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
args["destination"] = dst | |
err = c.CopyObject(dst, src) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed", err) | |
return | |
} | |
// Get copied object and check if content is equal | |
reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
decBytes, err = ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
if !bytes.Equal(decBytes, buf) { | |
logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) | |
return | |
} | |
reader.Close() | |
// Test in-place decryption. | |
dst, err = minio.NewDestinationInfo(bucketName, "srcObject", nil, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
args["destination"] = dst | |
src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) | |
args["source"] = src | |
err = c.CopyObject(dst, src) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err) | |
return | |
} | |
} | |
// Get copied decrypted object and check if content is equal | |
reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
defer reader.Close() | |
decBytes, err = ioutil.ReadAll(reader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
if !bytes.Equal(decBytes, buf) { | |
logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test encrypted copy object | |
func testUnencryptedToSSECCopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
var sseSrc encrypt.ServerSide | |
sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
// Test encrypted copy object | |
func testUnencryptedToSSES3CopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
var sseSrc encrypt.ServerSide | |
sseDst := encrypt.NewSSE() | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
// Test encrypted copy object | |
func testUnencryptedToUnencryptedCopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
var sseSrc, sseDst encrypt.ServerSide | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
// Test encrypted copy object | |
func testEncryptedSSECToSSECCopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) | |
sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
// Test encrypted copy object | |
func testEncryptedSSECToSSES3CopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) | |
sseDst := encrypt.NewSSE() | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
// Test encrypted copy object | |
func testEncryptedSSECToUnencryptedCopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) | |
var sseDst encrypt.ServerSide | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
// Test encrypted copy object | |
func testEncryptedSSES3ToSSECCopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
sseSrc := encrypt.NewSSE() | |
sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
// Test encrypted copy object | |
func testEncryptedSSES3ToSSES3CopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
sseSrc := encrypt.NewSSE() | |
sseDst := encrypt.NewSSE() | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
// Test encrypted copy object | |
func testEncryptedSSES3ToUnencryptedCopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
sseSrc := encrypt.NewSSE() | |
var sseDst encrypt.ServerSide | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
// Test encrypted copy object | |
func testEncryptedCopyObjectV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) | |
sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) | |
// c.TraceOn(os.Stderr) | |
testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) | |
} | |
func testDecryptedCopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) | |
return | |
} | |
bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object" | |
if err = c.MakeBucket(bucketName, "us-east-1"); err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)) | |
_, err = c.PutObject(bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{ | |
ServerSideEncryption: encryption, | |
}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
return | |
} | |
src := minio.NewSourceInfo(bucketName, objectName, encrypt.SSECopy(encryption)) | |
args["source"] = src | |
dst, err := minio.NewDestinationInfo(bucketName, "decrypted-"+objectName, nil, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
args["destination"] = dst | |
if err = c.CopyObject(dst, src); err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed", err) | |
return | |
} | |
if _, err = c.GetObject(bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testSSECMultipartEncryptedToSSECCopyObjectPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 6MB of data | |
buf := bytes.Repeat([]byte("abcdef"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
password := "correct horse battery staple" | |
srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) | |
// Upload a 6MB object using multipart mechanism | |
uploadID, err := c.NewMultipartUpload(bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
var completeParts []minio.CompletePart | |
part, err := c.PutObjectPart(bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) | |
} | |
completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) | |
part, err = c.PutObjectPart(bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) | |
} | |
completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(bucketName, objectName, uploadID, completeParts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) | |
uploadID, err = c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
encrypt.SSECopy(srcencryption).Marshal(header) | |
dstencryption.Marshal(header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (6*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} | |
getOpts.SetRange(0, 6*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 6*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 6MB", err) | |
} | |
getOpts.SetRange(6*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 6*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:6*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 6MB", err) | |
} | |
if getBuf[6*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
// Test Core CopyObjectPart implementation | |
func testSSECEncryptedToSSECCopyObjectPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 5MB of data | |
buf := bytes.Repeat([]byte("abcde"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
password := "correct horse battery staple" | |
srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) | |
putmetadata := map[string]string{ | |
"Content-Type": "binary/octet-stream", | |
} | |
opts := minio.PutObjectOptions{ | |
UserMetadata: putmetadata, | |
ServerSideEncryption: srcencryption, | |
} | |
objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
} | |
if objInfo.Size != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) | |
uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
encrypt.SSECopy(srcencryption).Marshal(header) | |
dstencryption.Marshal(header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (5*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} | |
getOpts.SetRange(0, 5*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 5*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) | |
} | |
getOpts.SetRange(5*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 5*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:5*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) | |
} | |
if getBuf[5*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
// Test Core CopyObjectPart implementation for SSEC encrypted to unencrypted copy | |
func testSSECEncryptedToUnencryptedCopyPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 5MB of data | |
buf := bytes.Repeat([]byte("abcde"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
password := "correct horse battery staple" | |
srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) | |
opts := minio.PutObjectOptions{ | |
UserMetadata: map[string]string{ | |
"Content-Type": "binary/octet-stream", | |
}, | |
ServerSideEncryption: srcencryption, | |
} | |
objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
} | |
if objInfo.Size != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
var dstencryption encrypt.ServerSide | |
uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
encrypt.SSECopy(srcencryption).Marshal(header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (5*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{} | |
getOpts.SetRange(0, 5*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 5*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) | |
} | |
getOpts.SetRange(5*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 5*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:5*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) | |
} | |
if getBuf[5*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
// Test Core CopyObjectPart implementation for SSEC encrypted to SSE-S3 encrypted copy | |
func testSSECEncryptedToSSES3CopyObjectPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 5MB of data | |
buf := bytes.Repeat([]byte("abcde"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
password := "correct horse battery staple" | |
srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) | |
putmetadata := map[string]string{ | |
"Content-Type": "binary/octet-stream", | |
} | |
opts := minio.PutObjectOptions{ | |
UserMetadata: putmetadata, | |
ServerSideEncryption: srcencryption, | |
} | |
objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
} | |
if objInfo.Size != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
dstencryption := encrypt.NewSSE() | |
uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
encrypt.SSECopy(srcencryption).Marshal(header) | |
dstencryption.Marshal(header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (5*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{} | |
getOpts.SetRange(0, 5*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 5*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) | |
} | |
getOpts.SetRange(5*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 5*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:5*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) | |
} | |
if getBuf[5*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
// Test Core CopyObjectPart implementation for unencrypted to SSEC encryption copy part | |
func testUnencryptedToSSECCopyObjectPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 5MB of data | |
buf := bytes.Repeat([]byte("abcde"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
password := "correct horse battery staple" | |
putmetadata := map[string]string{ | |
"Content-Type": "binary/octet-stream", | |
} | |
opts := minio.PutObjectOptions{ | |
UserMetadata: putmetadata, | |
} | |
objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
} | |
if objInfo.Size != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) | |
uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
dstencryption.Marshal(header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (5*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} | |
getOpts.SetRange(0, 5*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 5*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) | |
} | |
getOpts.SetRange(5*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 5*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:5*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) | |
} | |
if getBuf[5*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
// Test Core CopyObjectPart implementation for unencrypted to unencrypted copy | |
func testUnencryptedToUnencryptedCopyPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 5MB of data | |
buf := bytes.Repeat([]byte("abcde"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
putmetadata := map[string]string{ | |
"Content-Type": "binary/octet-stream", | |
} | |
opts := minio.PutObjectOptions{ | |
UserMetadata: putmetadata, | |
} | |
objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
} | |
if objInfo.Size != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (5*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{} | |
getOpts.SetRange(0, 5*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 5*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) | |
} | |
getOpts.SetRange(5*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 5*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:5*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) | |
} | |
if getBuf[5*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
// Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy | |
func testUnencryptedToSSES3CopyObjectPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 5MB of data | |
buf := bytes.Repeat([]byte("abcde"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
opts := minio.PutObjectOptions{ | |
UserMetadata: map[string]string{ | |
"Content-Type": "binary/octet-stream", | |
}, | |
} | |
objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
} | |
if objInfo.Size != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
dstencryption := encrypt.NewSSE() | |
uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
dstencryption.Marshal(header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (5*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{} | |
getOpts.SetRange(0, 5*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 5*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) | |
} | |
getOpts.SetRange(5*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 5*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:5*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) | |
} | |
if getBuf[5*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
// Test Core CopyObjectPart implementation for SSE-S3 to SSEC encryption copy part | |
func testSSES3EncryptedToSSECCopyObjectPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 5MB of data | |
buf := bytes.Repeat([]byte("abcde"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
password := "correct horse battery staple" | |
srcEncryption := encrypt.NewSSE() | |
opts := minio.PutObjectOptions{ | |
UserMetadata: map[string]string{ | |
"Content-Type": "binary/octet-stream", | |
}, | |
ServerSideEncryption: srcEncryption, | |
} | |
objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
} | |
if objInfo.Size != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) | |
uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
dstencryption.Marshal(header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (5*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} | |
getOpts.SetRange(0, 5*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 5*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) | |
} | |
getOpts.SetRange(5*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 5*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:5*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) | |
} | |
if getBuf[5*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
// Test Core CopyObjectPart implementation for unencrypted to unencrypted copy | |
func testSSES3EncryptedToUnencryptedCopyPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 5MB of data | |
buf := bytes.Repeat([]byte("abcde"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
srcEncryption := encrypt.NewSSE() | |
opts := minio.PutObjectOptions{ | |
UserMetadata: map[string]string{ | |
"Content-Type": "binary/octet-stream", | |
}, | |
ServerSideEncryption: srcEncryption, | |
} | |
objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
} | |
if objInfo.Size != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (5*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{} | |
getOpts.SetRange(0, 5*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 5*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) | |
} | |
getOpts.SetRange(5*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 5*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:5*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) | |
} | |
if getBuf[5*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
// Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy | |
func testSSES3EncryptedToSSES3CopyObjectPart() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObjectPart(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
client, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Instantiate new core client object. | |
c := minio.Core{client} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
} | |
defer cleanupBucket(bucketName, client) | |
// Make a buffer with 5MB of data | |
buf := bytes.Repeat([]byte("abcde"), 1024*1024) | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
srcEncryption := encrypt.NewSSE() | |
opts := minio.PutObjectOptions{ | |
UserMetadata: map[string]string{ | |
"Content-Type": "binary/octet-stream", | |
}, | |
ServerSideEncryption: srcEncryption, | |
} | |
objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
} | |
if objInfo.Size != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) | |
} | |
destBucketName := bucketName | |
destObjectName := objectName + "-dest" | |
dstencryption := encrypt.NewSSE() | |
uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) | |
} | |
// Content of the destination object will be two copies of | |
// `objectName` concatenated, followed by first byte of | |
// `objectName`. | |
metadata := make(map[string]string) | |
header := make(http.Header) | |
dstencryption.Marshal(header) | |
for k, v := range header { | |
metadata[k] = v[0] | |
} | |
metadata["x-amz-copy-source-if-match"] = objInfo.ETag | |
// First of three parts | |
fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Second of three parts | |
sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Last of three parts | |
lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) | |
} | |
// Complete the multipart upload | |
_, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) | |
} | |
// Stat the object and check its length matches | |
objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "StatObject call failed", err) | |
} | |
if objInfo.Size != (5*1024*1024)*2+1 { | |
logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) | |
} | |
// Now we read the data back | |
getOpts := minio.GetObjectOptions{} | |
getOpts.SetRange(0, 5*1024*1024-1) | |
r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf := make([]byte, 5*1024*1024) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf, buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) | |
} | |
getOpts.SetRange(5*1024*1024, 0) | |
r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject call failed", err) | |
} | |
getBuf = make([]byte, 5*1024*1024+1) | |
_, err = io.ReadFull(r, getBuf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Read buffer failed", err) | |
} | |
if !bytes.Equal(getBuf[:5*1024*1024], buf) { | |
logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) | |
} | |
if getBuf[5*1024*1024] != buf[0] { | |
logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) | |
} | |
successLogger(testName, function, args, startTime).Info() | |
// Do not need to remove destBucketName its same as bucketName. | |
} | |
func testUserMetadataCopying() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
// c.TraceOn(os.Stderr) | |
testUserMetadataCopyingWrapper(c) | |
} | |
func testUserMetadataCopyingWrapper(c *minio.Client) { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err := c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
fetchMeta := func(object string) (h http.Header) { | |
objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
h = make(http.Header) | |
for k, vs := range objInfo.Metadata { | |
if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { | |
for _, v := range vs { | |
h.Add(k, v) | |
} | |
} | |
} | |
return h | |
} | |
// 1. create a client encrypted object to copy by uploading | |
const srcSize = 1024 * 1024 | |
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB | |
metadata := make(http.Header) | |
metadata.Set("x-amz-meta-myheader", "myvalue") | |
m := make(map[string]string) | |
m["x-amz-meta-myheader"] = "myvalue" | |
_, err = c.PutObject(bucketName, "srcObject", | |
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err) | |
return | |
} | |
if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { | |
logError(testName, function, args, startTime, "", "Metadata match failed", err) | |
return | |
} | |
// 2. create source | |
src := minio.NewSourceInfo(bucketName, "srcObject", nil) | |
// 2.1 create destination with metadata set | |
dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", nil, map[string]string{"notmyheader": "notmyvalue"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
// 3. Check that copying to an object with metadata set resets | |
// the headers on the copy. | |
args["source"] = src | |
args["destination"] = dst1 | |
err = c.CopyObject(dst1, src) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed", err) | |
return | |
} | |
expectedHeaders := make(http.Header) | |
expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") | |
if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { | |
logError(testName, function, args, startTime, "", "Metadata match failed", err) | |
return | |
} | |
// 4. create destination with no metadata set and same source | |
dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", nil, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
src = minio.NewSourceInfo(bucketName, "srcObject", nil) | |
// 5. Check that copying to an object with no metadata set, | |
// copies metadata. | |
args["source"] = src | |
args["destination"] = dst2 | |
err = c.CopyObject(dst2, src) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed", err) | |
return | |
} | |
expectedHeaders = metadata | |
if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { | |
logError(testName, function, args, startTime, "", "Metadata match failed", err) | |
return | |
} | |
// 6. Compose a pair of sources. | |
srcs := []minio.SourceInfo{ | |
minio.NewSourceInfo(bucketName, "srcObject", nil), | |
minio.NewSourceInfo(bucketName, "srcObject", nil), | |
} | |
dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", nil, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
function = "ComposeObject(destination, sources)" | |
args["source"] = srcs | |
args["destination"] = dst3 | |
err = c.ComposeObject(dst3, srcs) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ComposeObject failed", err) | |
return | |
} | |
// Check that no headers are copied in this case | |
if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { | |
logError(testName, function, args, startTime, "", "Metadata match failed", err) | |
return | |
} | |
// 7. Compose a pair of sources with dest user metadata set. | |
srcs = []minio.SourceInfo{ | |
minio.NewSourceInfo(bucketName, "srcObject", nil), | |
minio.NewSourceInfo(bucketName, "srcObject", nil), | |
} | |
dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) | |
return | |
} | |
function = "ComposeObject(destination, sources)" | |
args["source"] = srcs | |
args["destination"] = dst4 | |
err = c.ComposeObject(dst4, srcs) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ComposeObject failed", err) | |
return | |
} | |
// Check that no headers are copied in this case | |
expectedHeaders = make(http.Header) | |
expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") | |
if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { | |
logError(testName, function, args, startTime, "", "Metadata match failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testUserMetadataCopyingV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "CopyObject(destination, source)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) | |
return | |
} | |
// c.TraceOn(os.Stderr) | |
testUserMetadataCopyingWrapper(c) | |
} | |
func testStorageClassMetadataPutObject() { | |
// initialize logging params | |
startTime := time.Now() | |
function := "testStorageClassMetadataPutObject()" | |
args := map[string]interface{}{} | |
testName := getFuncName() | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
fetchMeta := func(object string) (h http.Header) { | |
objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
h = make(http.Header) | |
for k, vs := range objInfo.Metadata { | |
if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { | |
for _, v := range vs { | |
h.Add(k, v) | |
} | |
} | |
} | |
return h | |
} | |
metadata := make(http.Header) | |
metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") | |
emptyMetadata := make(http.Header) | |
const srcSize = 1024 * 1024 | |
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB | |
_, err = c.PutObject(bucketName, "srcObjectRRSClass", | |
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
// Get the returned metadata | |
returnedMeta := fetchMeta("srcObjectRRSClass") | |
// The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) | |
if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { | |
logError(testName, function, args, startTime, "", "Metadata match failed", err) | |
return | |
} | |
metadata = make(http.Header) | |
metadata.Set("x-amz-storage-class", "STANDARD") | |
_, err = c.PutObject(bucketName, "srcObjectSSClass", | |
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) { | |
logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testStorageClassInvalidMetadataPutObject() { | |
// initialize logging params | |
startTime := time.Now() | |
function := "testStorageClassInvalidMetadataPutObject()" | |
args := map[string]interface{}{} | |
testName := getFuncName() | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
const srcSize = 1024 * 1024 | |
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB | |
_, err = c.PutObject(bucketName, "srcObjectRRSClass", | |
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"}) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
func testStorageClassMetadataCopyObject() { | |
// initialize logging params | |
startTime := time.Now() | |
function := "testStorageClassMetadataCopyObject()" | |
args := map[string]interface{}{} | |
testName := getFuncName() | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) | |
return | |
} | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") | |
// Make a new bucket in 'us-east-1' (source bucket). | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
fetchMeta := func(object string) (h http.Header) { | |
objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) | |
args["bucket"] = bucketName | |
args["object"] = object | |
if err != nil { | |
logError(testName, function, args, startTime, "", "Stat failed", err) | |
return | |
} | |
h = make(http.Header) | |
for k, vs := range objInfo.Metadata { | |
if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { | |
for _, v := range vs { | |
h.Add(k, v) | |
} | |
} | |
} | |
return h | |
} | |
metadata := make(http.Header) | |
metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") | |
emptyMetadata := make(http.Header) | |
const srcSize = 1024 * 1024 | |
buf := bytes.Repeat([]byte("abcde"), srcSize) | |
// Put an object with RRS Storage class | |
_, err = c.PutObject(bucketName, "srcObjectRRSClass", | |
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
// Make server side copy of object uploaded in previous step | |
src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) | |
dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) | |
if err = c.CopyObject(dst, src); err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err) | |
} | |
// Get the returned metadata | |
returnedMeta := fetchMeta("srcObjectRRSClassCopy") | |
// The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) | |
if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { | |
logError(testName, function, args, startTime, "", "Metadata match failed", err) | |
return | |
} | |
metadata = make(http.Header) | |
metadata.Set("x-amz-storage-class", "STANDARD") | |
// Put an object with Standard Storage class | |
_, err = c.PutObject(bucketName, "srcObjectSSClass", | |
bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
// Make server side copy of object uploaded in previous step | |
src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) | |
dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", nil, nil) | |
if err = c.CopyObject(dst, src); err != nil { | |
logError(testName, function, args, startTime, "", "CopyObject failed on SS", err) | |
} | |
// Fetch the meta data of copied object | |
if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) { | |
logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test put object with size -1 byte object. | |
func testPutObjectNoLengthV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, reader, size, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"size": -1, | |
"opts": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
objectName := bucketName + "unique" | |
args["objectName"] = objectName | |
bufSize := dataFileMap["datafile-129-MB"] | |
var reader = getDataReader("datafile-129-MB") | |
defer reader.Close() | |
args["size"] = bufSize | |
// Upload an object. | |
n, err := c.PutObject(bucketName, objectName, reader, -1, minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) | |
return | |
} | |
if n != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(n), err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test put objects of unknown size. | |
func testPutObjectsUnknownV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, reader,size,opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"size": "", | |
"opts": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Issues are revealed by trying to upload multiple files of unknown size | |
// sequentially (on 4GB machines) | |
for i := 1; i <= 4; i++ { | |
// Simulate that we could be receiving byte slices of data that we want | |
// to upload as a file | |
rpipe, wpipe := io.Pipe() | |
defer rpipe.Close() | |
go func() { | |
b := []byte("test") | |
wpipe.Write(b) | |
wpipe.Close() | |
}() | |
// Upload the object. | |
objectName := fmt.Sprintf("%sunique%d", bucketName, i) | |
args["objectName"] = objectName | |
n, err := c.PutObject(bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) | |
return | |
} | |
args["size"] = n | |
if n != int64(4) { | |
logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(n), err) | |
return | |
} | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test put object with 0 byte object. | |
func testPutObject0ByteV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObject(bucketName, objectName, reader, size, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectName": "", | |
"size": 0, | |
"opts": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
objectName := bucketName + "unique" | |
args["objectName"] = objectName | |
args["opts"] = minio.PutObjectOptions{} | |
// Upload an object. | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) | |
return | |
} | |
if n != 0 { | |
logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(n), err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test expected error cases | |
func testComposeObjectErrorCases() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "ComposeObject(destination, sourceList)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
testComposeObjectErrorCasesWrapper(c) | |
} | |
// Test concatenating 10K objects | |
func testCompose10KSources() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "ComposeObject(destination, sourceList)" | |
args := map[string]interface{}{} | |
// Instantiate new minio client object | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) | |
return | |
} | |
testComposeMultipleSources(c) | |
} | |
// Tests comprehensive list of all methods. | |
func testFunctionalV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "testFunctionalV2()" | |
functionAll := "" | |
args := map[string]interface{}{} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) | |
return | |
} | |
// Enable to debug | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
location := "us-east-1" | |
// Make a new bucket. | |
function = "MakeBucket(bucketName, location)" | |
functionAll = "MakeBucket(bucketName, location)" | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"location": location, | |
} | |
err = c.MakeBucket(bucketName, location) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
// Generate a random file name. | |
fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
file, err := os.Create(fileName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "file create failed", err) | |
return | |
} | |
for i := 0; i < 3; i++ { | |
buf := make([]byte, rand.Intn(1<<19)) | |
_, err = file.Write(buf) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "file write failed", err) | |
return | |
} | |
} | |
file.Close() | |
// Verify if bucket exits and you have access. | |
var exists bool | |
function = "BucketExists(bucketName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
} | |
exists, err = c.BucketExists(bucketName) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "BucketExists failed", err) | |
return | |
} | |
if !exists { | |
logError(testName, function, args, startTime, "", "Could not find existing bucket "+bucketName, err) | |
return | |
} | |
// Make the bucket 'public read/write'. | |
function = "SetBucketPolicy(bucketName, bucketPolicy)" | |
functionAll += ", " + function | |
readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}` | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"bucketPolicy": readWritePolicy, | |
} | |
err = c.SetBucketPolicy(bucketName, readWritePolicy) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) | |
return | |
} | |
// List all buckets. | |
function = "ListBuckets()" | |
functionAll += ", " + function | |
args = nil | |
buckets, err := c.ListBuckets() | |
if len(buckets) == 0 { | |
logError(testName, function, args, startTime, "", "List buckets cannot be empty", err) | |
return | |
} | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ListBuckets failed", err) | |
return | |
} | |
// Verify if previously created bucket is listed in list buckets. | |
bucketFound := false | |
for _, bucket := range buckets { | |
if bucket.Name == bucketName { | |
bucketFound = true | |
} | |
} | |
// If bucket not found error out. | |
if !bucketFound { | |
logError(testName, function, args, startTime, "", "Bucket "+bucketName+"not found", err) | |
return | |
} | |
objectName := bucketName + "unique" | |
// Generate data | |
buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"contentType": "", | |
} | |
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) | |
return | |
} | |
objectNameNoLength := objectName + "-nolength" | |
args["objectName"] = objectNameNoLength | |
n, err = c.PutObject(bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
if n != int64(len(buf)) { | |
logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) | |
return | |
} | |
// Instantiate a done channel to close all listing. | |
doneCh := make(chan struct{}) | |
defer close(doneCh) | |
objFound := false | |
isRecursive := true // Recursive is true. | |
function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"isRecursive": isRecursive, | |
} | |
for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { | |
if obj.Key == objectName { | |
objFound = true | |
break | |
} | |
} | |
if !objFound { | |
logError(testName, function, args, startTime, "", "Could not find existing object "+objectName, err) | |
return | |
} | |
incompObjNotFound := true | |
function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"isRecursive": isRecursive, | |
} | |
for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { | |
if objIncompl.Key != "" { | |
incompObjNotFound = false | |
break | |
} | |
} | |
if !incompObjNotFound { | |
logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) | |
return | |
} | |
function = "GetObject(bucketName, objectName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
} | |
newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
newReadBytes, err := ioutil.ReadAll(newReader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
newReader.Close() | |
if !bytes.Equal(newReadBytes, buf) { | |
logError(testName, function, args, startTime, "", "Bytes mismatch", err) | |
return | |
} | |
function = "FGetObject(bucketName, objectName, fileName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"fileName": fileName + "-f", | |
} | |
err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FgetObject failed", err) | |
return | |
} | |
// Generate presigned HEAD object url. | |
function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"expires": 3600 * time.Second, | |
} | |
presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) | |
return | |
} | |
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "DefaultTransport failed", err) | |
return | |
} | |
httpClient := &http.Client{ | |
// Setting a sensible time out of 30secs to wait for response | |
// headers. Request is pro-actively canceled after 30secs | |
// with no response. | |
Timeout: 30 * time.Second, | |
Transport: transport, | |
} | |
req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) | |
return | |
} | |
// Verify if presigned url works. | |
resp, err := httpClient.Do(req) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) | |
return | |
} | |
if resp.StatusCode != http.StatusOK { | |
logError(testName, function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err) | |
return | |
} | |
if resp.Header.Get("ETag") == "" { | |
logError(testName, function, args, startTime, "", "Got empty ETag", err) | |
return | |
} | |
resp.Body.Close() | |
// Generate presigned GET object url. | |
function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName, | |
"expires": 3600 * time.Second, | |
} | |
presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) | |
return | |
} | |
// Verify if presigned url works. | |
req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) | |
return | |
} | |
resp, err = httpClient.Do(req) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) | |
return | |
} | |
if resp.StatusCode != http.StatusOK { | |
logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) | |
return | |
} | |
newPresignedBytes, err := ioutil.ReadAll(resp.Body) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
resp.Body.Close() | |
if !bytes.Equal(newPresignedBytes, buf) { | |
logError(testName, function, args, startTime, "", "Bytes mismatch", err) | |
return | |
} | |
// Set request parameters. | |
reqParams := make(url.Values) | |
reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") | |
// Generate presigned GET object url. | |
args["reqParams"] = reqParams | |
presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) | |
return | |
} | |
// Verify if presigned url works. | |
req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) | |
return | |
} | |
resp, err = httpClient.Do(req) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) | |
return | |
} | |
if resp.StatusCode != http.StatusOK { | |
logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) | |
return | |
} | |
newPresignedBytes, err = ioutil.ReadAll(resp.Body) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
if !bytes.Equal(newPresignedBytes, buf) { | |
logError(testName, function, args, startTime, "", "Bytes mismatch", err) | |
return | |
} | |
// Verify content disposition. | |
if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { | |
logError(testName, function, args, startTime, "", "wrong Content-Disposition received ", err) | |
return | |
} | |
function = "PresignedPutObject(bucketName, objectName, expires)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName + "-presigned", | |
"expires": 3600 * time.Second, | |
} | |
presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) | |
return | |
} | |
// Generate data more than 32K | |
buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) | |
req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) | |
return | |
} | |
resp, err = httpClient.Do(req) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) | |
return | |
} | |
function = "GetObject(bucketName, objectName)" | |
functionAll += ", " + function | |
args = map[string]interface{}{ | |
"bucketName": bucketName, | |
"objectName": objectName + "-presigned", | |
} | |
newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObject failed", err) | |
return | |
} | |
newReadBytes, err = ioutil.ReadAll(newReader) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "ReadAll failed", err) | |
return | |
} | |
newReader.Close() | |
if !bytes.Equal(newReadBytes, buf) { | |
logError(testName, function, args, startTime, "", "Bytes mismatch", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
os.Remove(fileName) | |
os.Remove(fileName + "-f") | |
successLogger(testName, functionAll, args, startTime).Info() | |
} | |
// Test get object with GetObjectWithContext | |
func testGetObjectWithContext() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObjectWithContext(ctx, bucketName, objectName)" | |
args := map[string]interface{}{ | |
"ctx": "", | |
"bucketName": "", | |
"objectName": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
_, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) | |
args["ctx"] = ctx | |
defer cancel() | |
r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) | |
return | |
} | |
if _, err = r.Stat(); err == nil { | |
logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) | |
return | |
} | |
r.Close() | |
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) | |
args["ctx"] = ctx | |
defer cancel() | |
// Read the data back | |
r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObjectWithContext failed", err) | |
return | |
} | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "object Stat call failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err) | |
return | |
} | |
if err := r.Close(); err != nil { | |
logError(testName, function, args, startTime, "", "object Close() call failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test get object with FGetObjectWithContext | |
func testFGetObjectWithContext() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "FGetObjectWithContext(ctx, bucketName, objectName, fileName)" | |
args := map[string]interface{}{ | |
"ctx": "", | |
"bucketName": "", | |
"objectName": "", | |
"fileName": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
bufSize := dataFileMap["datafile-1-MB"] | |
var reader = getDataReader("datafile-1-MB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
_, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) | |
args["ctx"] = ctx | |
defer cancel() | |
fileName := "tempfile-context" | |
args["fileName"] = fileName | |
// Read the data back | |
err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) | |
return | |
} | |
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) | |
defer cancel() | |
// Read the data back | |
err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FGetObjectWithContext with long timeout failed", err) | |
return | |
} | |
if err = os.Remove(fileName + "-fcontext"); err != nil { | |
logError(testName, function, args, startTime, "", "Remove file failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test get object ACLs with GetObjectACLWithContext | |
func testGetObjectACLWithContext() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObjectACLWithContext(ctx, bucketName, objectName)" | |
args := map[string]interface{}{ | |
"ctx": "", | |
"bucketName": "", | |
"objectName": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// skipping region functional tests for non s3 runs | |
if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { | |
ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() | |
return | |
} | |
// Instantiate new minio client object. | |
c, err := minio.NewV4( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
bufSize := dataFileMap["datafile-1-MB"] | |
var reader = getDataReader("datafile-1-MB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Add meta data to add a canned acl | |
metaData := map[string]string{ | |
"X-Amz-Acl": "public-read-write", | |
} | |
_, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) | |
args["ctx"] = ctx | |
defer cancel() | |
// Read the data back | |
objectInfo, getObjectACLErr := c.GetObjectACLWithContext(ctx, bucketName, objectName) | |
if getObjectACLErr == nil { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail", getObjectACLErr) | |
return | |
} | |
s, ok := objectInfo.Metadata["X-Amz-Acl"] | |
if !ok { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Acl\"", nil) | |
return | |
} | |
if len(s) != 1 { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) | |
return | |
} | |
if s[0] != "public-read-write" { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) | |
return | |
} | |
bufSize = dataFileMap["datafile-1-MB"] | |
var reader2 = getDataReader("datafile-1-MB") | |
defer reader2.Close() | |
// Save the data | |
objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Add meta data to add a canned acl | |
metaData = map[string]string{ | |
"X-Amz-Grant-Read": "id=fooread@minio.go", | |
"X-Amz-Grant-Write": "id=foowrite@minio.go", | |
} | |
_, err = c.PutObject(bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject failed", err) | |
return | |
} | |
ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) | |
args["ctx"] = ctx | |
defer cancel() | |
// Read the data back | |
objectInfo, getObjectACLErr = c.GetObjectACLWithContext(ctx, bucketName, objectName) | |
if getObjectACLErr == nil { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail", getObjectACLErr) | |
return | |
} | |
if len(objectInfo.Metadata) != 3 { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) | |
return | |
} | |
s, ok = objectInfo.Metadata["X-Amz-Grant-Read"] | |
if !ok { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Grant-Read\"", nil) | |
return | |
} | |
if len(s) != 1 { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) | |
return | |
} | |
if s[0] != "fooread@minio.go" { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) | |
return | |
} | |
s, ok = objectInfo.Metadata["X-Amz-Grant-Write"] | |
if !ok { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Grant-Write\"", nil) | |
return | |
} | |
if len(s) != 1 { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) | |
return | |
} | |
if s[0] != "foowrite@minio.go" { | |
logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test validates putObject with context to see if request cancellation is honored for V2. | |
func testPutObjectWithContextV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "PutObjectWithContext(ctx, bucketName, objectName, reader, size, opts)" | |
args := map[string]interface{}{ | |
"ctx": "", | |
"bucketName": "", | |
"objectName": "", | |
"size": "", | |
"opts": "", | |
} | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Make a new bucket. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
defer c.RemoveBucket(bucketName) | |
bufSize := dataFileMap["datatfile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) | |
args["objectName"] = objectName | |
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) | |
args["ctx"] = ctx | |
args["size"] = bufSize | |
defer cancel() | |
_, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectWithContext with short timeout failed", err) | |
return | |
} | |
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) | |
args["ctx"] = ctx | |
defer cancel() | |
reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
_, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test get object with GetObjectWithContext | |
func testGetObjectWithContextV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "GetObjectWithContext(ctx, bucketName, objectName)" | |
args := map[string]interface{}{ | |
"ctx": "", | |
"bucketName": "", | |
"objectName": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
_, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
return | |
} | |
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) | |
args["ctx"] = ctx | |
defer cancel() | |
r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) | |
return | |
} | |
if _, err = r.Stat(); err == nil { | |
logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) | |
return | |
} | |
r.Close() | |
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) | |
defer cancel() | |
// Read the data back | |
r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "GetObjectWithContext shouldn't fail on longer timeout", err) | |
return | |
} | |
st, err := r.Stat() | |
if err != nil { | |
logError(testName, function, args, startTime, "", "object Stat call failed", err) | |
return | |
} | |
if st.Size != int64(bufSize) { | |
logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err) | |
return | |
} | |
if err := r.Close(); err != nil { | |
logError(testName, function, args, startTime, "", " object Close() call failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test get object with FGetObjectWithContext | |
func testFGetObjectWithContextV2() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "FGetObjectWithContext(ctx, bucketName, objectName,fileName)" | |
args := map[string]interface{}{ | |
"ctx": "", | |
"bucketName": "", | |
"objectName": "", | |
"fileName": "", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.NewV2( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket call failed", err) | |
return | |
} | |
bufSize := dataFileMap["datatfile-1-MB"] | |
var reader = getDataReader("datafile-1-MB") | |
defer reader.Close() | |
// Save the data | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
_, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "PutObject call failed", err) | |
return | |
} | |
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) | |
args["ctx"] = ctx | |
defer cancel() | |
fileName := "tempfile-context" | |
args["fileName"] = fileName | |
// Read the data back | |
err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) | |
if err == nil { | |
logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) | |
return | |
} | |
ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) | |
defer cancel() | |
// Read the data back | |
err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "FGetObjectWithContext call shouldn't fail on long timeout", err) | |
return | |
} | |
if err = os.Remove(fileName + "-fcontext"); err != nil { | |
logError(testName, function, args, startTime, "", "Remove file failed", err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test list object v1 and V2 | |
func testListObjects() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectPrefix": "", | |
"recursive": "true", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
// Make a new bucket. | |
err = c.MakeBucket(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
testObjects := []struct { | |
name string | |
storageClass string | |
}{ | |
// Special characters | |
{"foo bar", "STANDARD"}, | |
{"foo-%", "STANDARD"}, | |
{"random-object-1", "STANDARD"}, | |
{"random-object-2", "REDUCED_REDUNDANCY"}, | |
} | |
for i, object := range testObjects { | |
bufSize := dataFileMap["datafile-33-kB"] | |
var reader = getDataReader("datafile-33-kB") | |
defer reader.Close() | |
_, err = c.PutObject(bucketName, object.name, reader, int64(bufSize), | |
minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass}) | |
if err != nil { | |
logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err) | |
return | |
} | |
} | |
testList := func(listFn func(string, string, bool, <-chan struct{}) <-chan minio.ObjectInfo, bucket string) { | |
// Create a done channel to control 'ListObjects' go routine. | |
doneCh := make(chan struct{}) | |
// Exit cleanly upon return. | |
defer close(doneCh) | |
var objCursor int | |
// check for object name and storage-class from listing object result | |
for objInfo := range listFn(bucket, "", true, doneCh) { | |
if objInfo.Err != nil { | |
logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) | |
return | |
} | |
if objInfo.Key != testObjects[objCursor].name { | |
logError(testName, function, args, startTime, "", "ListObjects does not return expected object name", err) | |
} | |
if objInfo.StorageClass != testObjects[objCursor].storageClass { | |
// Ignored as Gateways (Azure/GCS etc) wont return storage class | |
ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() | |
} | |
objCursor++ | |
} | |
if objCursor != len(testObjects) { | |
logError(testName, function, args, startTime, "", "ListObjects returned unexpected number of items", errors.New("")) | |
} | |
} | |
testList(c.ListObjects, bucketName) | |
testList(c.ListObjectsV2, bucketName) | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Test deleting multiple objects with object retention set in Governance mode | |
func testRemoveObjectsWithOptions() { | |
// initialize logging params | |
startTime := time.Now() | |
testName := getFuncName() | |
function := "RemoveObjectsWithOptions(bucketName, objectsCh, opts)" | |
args := map[string]interface{}{ | |
"bucketName": "", | |
"objectPrefix": "", | |
"recursive": "true", | |
} | |
// Seed random based on current time. | |
rand.Seed(time.Now().Unix()) | |
// Instantiate new minio client object. | |
c, err := minio.New( | |
os.Getenv(serverEndpoint), | |
os.Getenv(accessKey), | |
os.Getenv(secretKey), | |
mustParseBool(os.Getenv(enableHTTPS)), | |
) | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) | |
return | |
} | |
// Enable tracing, write to stderr. | |
// c.TraceOn(os.Stderr) | |
// Set user agent. | |
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") | |
// Generate a new random bucket name. | |
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") | |
args["bucketName"] = bucketName | |
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") | |
args["objectName"] = objectName | |
// Make a new bucket. | |
err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") | |
if err != nil { | |
logError(testName, function, args, startTime, "", "MakeBucket failed", err) | |
return | |
} | |
bufSize := dataFileMap["datafile-129-MB"] | |
var reader = getDataReader("datafile-129-MB") | |
defer reader.Close() | |
n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) | |
if err != nil { | |
log.Fatalln(err) | |
} | |
log.Println("Uploaded", objectName, " of size: ", n, "to bucket: ", bucketName, "Successfully.") | |
t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC) | |
m := minio.RetentionMode(minio.Governance) | |
opts := minio.PutObjectRetentionOptions{ | |
GovernanceBypass: false, | |
RetainUntilDate: &t, | |
Mode: &m, | |
} | |
err = c.PutObjectRetention(bucketName, objectName, opts) | |
if err != nil { | |
log.Fatalln(err) | |
} | |
objectsCh := make(chan string) | |
// Send object names that are needed to be removed to objectsCh | |
go func() { | |
defer close(objectsCh) | |
// List all objects from a bucket-name with a matching prefix. | |
for object := range c.ListObjects(bucketName, "", true, nil) { | |
if object.Err != nil { | |
log.Fatalln(object.Err) | |
} | |
objectsCh <- object.Key | |
} | |
}() | |
for rErr := range c.RemoveObjects(bucketName, objectsCh) { | |
// Error is expected here because Retention is set on the object | |
// and RemoveObjects is called without Bypass Governance | |
if rErr.Err == nil { | |
logError(testName, function, args, startTime, "", "Expected error during deletion", nil) | |
return | |
} | |
} | |
objectsCh1 := make(chan string) | |
// Send object names that are needed to be removed to objectsCh | |
go func() { | |
defer close(objectsCh1) | |
// List all objects from a bucket-name with a matching prefix. | |
for object := range c.ListObjects(bucketName, "", true, nil) { | |
if object.Err != nil { | |
log.Fatalln(object.Err) | |
} | |
objectsCh1 <- object.Key | |
} | |
}() | |
opts1 := minio.RemoveObjectsOptions{ | |
GovernanceBypass: true, | |
} | |
for rErr := range c.RemoveObjectsWithOptions(bucketName, objectsCh1, opts1) { | |
// Error is not expected here because Retention is set on the object | |
// and RemoveObjects is called with Bypass Governance | |
logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err) | |
return | |
} | |
// Delete all objects and buckets | |
if err = cleanupBucket(bucketName, c); err != nil { | |
logError(testName, function, args, startTime, "", "Cleanup failed", err) | |
return | |
} | |
successLogger(testName, function, args, startTime).Info() | |
} | |
// Convert string to bool and always return false if any error | |
func mustParseBool(str string) bool { | |
b, err := strconv.ParseBool(str) | |
if err != nil { | |
return false | |
} | |
return b | |
} | |
func main() { | |
// Output to stdout instead of the default stderr | |
log.SetOutput(os.Stdout) | |
// create custom formatter | |
mintFormatter := mintJSONFormatter{} | |
// set custom formatter | |
log.SetFormatter(&mintFormatter) | |
// log Info or above -- success cases are Info level, failures are Fatal level | |
log.SetLevel(log.InfoLevel) | |
tls := mustParseBool(os.Getenv(enableHTTPS)) | |
kmsEnabled := mustParseBool(os.Getenv(enableKMS)) | |
// execute tests | |
if isFullMode() { | |
testMakeBucketErrorV2() | |
testGetObjectClosedTwiceV2() | |
testFPutObjectV2() | |
testMakeBucketRegionsV2() | |
testGetObjectReadSeekFunctionalV2() | |
testGetObjectReadAtFunctionalV2() | |
testCopyObjectV2() | |
testFunctionalV2() | |
testComposeObjectErrorCasesV2() | |
testCompose10KSourcesV2() | |
testUserMetadataCopyingV2() | |
testPutObject0ByteV2() | |
testPutObjectNoLengthV2() | |
testPutObjectsUnknownV2() | |
testGetObjectWithContextV2() | |
testFPutObjectWithContextV2() | |
testFGetObjectWithContextV2() | |
testPutObjectWithContextV2() | |
testMakeBucketError() | |
testMakeBucketRegions() | |
testPutObjectWithMetadata() | |
testPutObjectReadAt() | |
testPutObjectStreaming() | |
testGetObjectSeekEnd() | |
testGetObjectClosedTwice() | |
testRemoveMultipleObjects() | |
testFPutObjectMultipart() | |
testFPutObject() | |
testGetObjectReadSeekFunctional() | |
testGetObjectReadAtFunctional() | |
testGetObjectReadAtWhenEOFWasReached() | |
testPresignedPostPolicy() | |
testCopyObject() | |
testComposeObjectErrorCases() | |
testCompose10KSources() | |
testUserMetadataCopying() | |
testBucketNotification() | |
testFunctional() | |
testGetObjectModified() | |
testPutObjectUploadSeekedObject() | |
testGetObjectWithContext() | |
testFPutObjectWithContext() | |
testFGetObjectWithContext() | |
testGetObjectACLWithContext() | |
testPutObjectWithContext() | |
testStorageClassMetadataPutObject() | |
testStorageClassInvalidMetadataPutObject() | |
testStorageClassMetadataCopyObject() | |
testPutObjectWithContentLanguage() | |
testListObjects() | |
//testRemoveObjectsWithOptions() | |
testListObjectVersions() | |
testStatObjectWithVersioning() | |
testGetObjectWithVersioning() | |
testCopyObjectWithVersioning() | |
testComposeObjectWithVersioning() | |
testRemoveObjectWithVersioning() | |
testRemoveObjectsWithVersioning() | |
testObjectTaggingWithVersioning() | |
// SSE-C tests will only work over TLS connection. | |
if tls { | |
testSSECEncryptionPutGet() | |
testSSECEncryptionFPut() | |
testSSECEncryptedGetObjectReadAtFunctional() | |
testSSECEncryptedGetObjectReadSeekFunctional() | |
testEncryptedCopyObjectV2() | |
testEncryptedSSECToSSECCopyObject() | |
testEncryptedSSECToUnencryptedCopyObject() | |
testUnencryptedToSSECCopyObject() | |
testUnencryptedToUnencryptedCopyObject() | |
testEncryptedEmptyObject() | |
testDecryptedCopyObject() | |
testSSECEncryptedToSSECCopyObjectPart() | |
testSSECMultipartEncryptedToSSECCopyObjectPart() | |
testSSECEncryptedToUnencryptedCopyPart() | |
testUnencryptedToSSECCopyObjectPart() | |
testUnencryptedToUnencryptedCopyPart() | |
if kmsEnabled { | |
testSSES3EncryptionPutGet() | |
testSSES3EncryptionFPut() | |
testSSES3EncryptedGetObjectReadAtFunctional() | |
testSSES3EncryptedGetObjectReadSeekFunctional() | |
testEncryptedSSECToSSES3CopyObject() | |
testEncryptedSSES3ToSSECCopyObject() | |
testEncryptedSSES3ToSSES3CopyObject() | |
testEncryptedSSES3ToUnencryptedCopyObject() | |
testUnencryptedToSSES3CopyObject() | |
testSSECEncryptedToSSES3CopyObjectPart() | |
testUnencryptedToSSES3CopyObjectPart() | |
testSSES3EncryptedToSSECCopyObjectPart() | |
testSSES3EncryptedToUnencryptedCopyPart() | |
testSSES3EncryptedToSSES3CopyObjectPart() | |
} | |
} | |
} else { | |
testFunctional() | |
testFunctionalV2() | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment