Skip to content

Instantly share code, notes, and snippets.

@fils
Last active July 28, 2018 00:49
Show Gist options
  • Save fils/8c9989a23aafe338905697a48a8e6d65 to your computer and use it in GitHub Desktop.
Save fils/8c9989a23aafe338905697a48a8e6d65 to your computer and use it in GitHub Desktop.
A test to see if the google go-cloud package works with minio via the Amazon Go SDK
package main
import (
"context"
"flag"
"fmt"
"io/ioutil"
"log"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/google/go-cloud/blob"
"github.com/google/go-cloud/blob/gcsblob"
"github.com/google/go-cloud/blob/s3blob"
"github.com/google/go-cloud/gcp"
)
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Command upload saves files to blob storage on GCP and AWS.
func main() {
// Define our input.
cloud := flag.String("cloud", "", "Cloud storage to use")
bucketName := flag.String("bucket", "go-cloud-bucket", "Name of bucket")
flag.Parse()
if flag.NArg() != 1 {
log.Fatal("Failed to provide file to upload")
}
file := flag.Arg(0)
ctx := context.Background()
// Open a connection to the bucket.
b, err := setupBucket(context.Background(), *cloud, *bucketName)
if err != nil {
log.Fatalf("Failed to setup bucket: %s", err)
}
// Prepare the file for upload.
data, err := ioutil.ReadFile(file)
if err != nil {
log.Fatalf("Failed to read file: %s", err)
}
w, err := b.NewWriter(ctx, file, nil)
if err != nil {
log.Fatalf("Failed to obtain writer: %s", err)
}
_, err = w.Write(data)
if err != nil {
log.Fatalf("Failed to write to bucket: %s", err)
}
if err = w.Close(); err != nil {
log.Fatalf("Failed to close: %s", err)
}
}
// setupBucket creates a connection to a particular cloud provider's blob storage.
func setupBucket(ctx context.Context, cloud, bucket string) (*blob.Bucket, error) {
switch cloud {
case "aws":
return setupAWS(ctx, bucket)
case "gcp":
return setupGCP(ctx, bucket)
default:
return nil, fmt.Errorf("invalid cloud provider: %s", cloud)
}
}
// setupGCP creates a connection to Google Cloud Storage (GCS).
func setupGCP(ctx context.Context, bucket string) (*blob.Bucket, error) {
// DefaultCredentials assumes a user has logged in with gcloud.
// See here for more information:
// https://cloud.google.com/docs/authentication/getting-started
creds, err := gcp.DefaultCredentials(ctx)
if err != nil {
return nil, err
}
c, err := gcp.NewHTTPClient(gcp.DefaultTransport(), gcp.CredentialsTokenSource(creds))
if err != nil {
return nil, err
}
return gcsblob.OpenBucket(ctx, bucket, c)
}
// setupAWS creates a connection to Simple Cloud Storage Service (S3).
func setupAWS(ctx context.Context, bucket string) (*blob.Bucket, error) {
c := &aws.Config{
// Either hard-code the region or use AWS_REGION.
Region: aws.String("us-east-2"),
Endpoint: aws.String("http://localhost:9000"),
// Need `S3ForcePathStyle: true` and `DisableSSL : true` (thanks Minio slack!)
S3ForcePathStyle: aws.Bool(true),
DisableSSL: aws.Bool(true),
// credentials.NewEnvCredentials assumes two environment variables are
// present:
// 1. AWS_ACCESS_KEY_ID, and
// 2. AWS_SECRET_ACCESS_KEY.
Credentials: credentials.NewEnvCredentials(),
}
s := session.Must(session.NewSession(c))
return s3blob.OpenBucket(ctx, s, bucket)
}
@fils
Copy link
Author

fils commented Jul 27, 2018

I seem to get all the way to the close....

fils@xps:~/src/go/src/opencoredata.org/ocdGarden/S3/go-cloud-test$ go run main.go  -cloud aws -bucket provisium  gopher.png
2018/07/27 08:21:02 Failed to close: MalformedXML: The XML you provided was not well-formed or did not validate against our published schema.
        status code: 400, request id: 15453C67491F4216, host id:
exit status 1

@fils
Copy link
Author

fils commented Jul 27, 2018

Fixed after some advice from the Minio slack channel with

		S3ForcePathStyle: aws.Bool(true),
		DisableSSL:       aws.Bool(true),

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment