Navigation Menu

Skip to content

Instantly share code, notes, and snippets.

@Kugelschieber
Last active December 22, 2019 13:03
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Kugelschieber/2c1e0a988db4545f8eaf647d9e46c4b9 to your computer and use it in GitHub Desktop.
Save Kugelschieber/2c1e0a988db4545f8eaf647d9e46c4b9 to your computer and use it in GitHub Desktop.
This little Go application will copy everything from one Google Cloud Storage Bucket to a MinIO Bucket. You'll need a GCS credentials file and a MinIO access key + secret.
package main
import (
"cloud.google.com/go/storage"
"context"
"github.com/minio/minio-go"
"google.golang.org/api/iterator"
"log"
"os"
)
const(
minioEndpoint = "minio.my-domain.com"
minioAccessKeyID = ""
minioSecretAccessKey = ""
minioUseSSL = true
minioTargetBucket = "target-bucket"
gcsSourceBucket = "source-bucket"
)
func getGCSClient() *storage.Client {
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatalln(err)
}
return client
}
func getMinioClient() *minio.Client {
minioClient, err := minio.New(minioEndpoint, minioAccessKeyID, minioSecretAccessKey, minioUseSSL)
if err != nil {
log.Fatalln(err)
}
return minioClient
}
func getGCSBucketObjectNames(bucket *storage.BucketHandle) []string {
var names []string
ctx := context.Background()
it := bucket.Objects(ctx, &storage.Query{Prefix: ""})
for {
attrs, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
log.Fatalln(err)
}
names = append(names, attrs.Name)
}
return names
}
func copyFile(bucket *storage.BucketHandle, minioClient *minio.Client, name string) {
log.Printf("Copying object: %s", name)
ctx := context.Background()
reader, err := bucket.Object(name).NewReader(ctx)
if err != nil {
log.Fatalln(err)
}
defer reader.Close()
_, err = minioClient.PutObject(minioTargetBucket, name, reader, -1, minio.PutObjectOptions{})
if err != nil {
log.Fatalln(err)
}
}
func main() {
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./gcs_credentials.json")
gcsClient := getGCSClient()
gcsBucket := gcsClient.Bucket(gcsSourceBucket)
gcsObjectNames := getGCSBucketObjectNames(gcsBucket)
minioClient := getMinioClient()
for _, name := range gcsObjectNames {
copyFile(gcsBucket, minioClient, name)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment