main.go

package main

import (
	"crypto/tls"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/endpoints"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/pkg/errors"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
)

var (
	regionKey                = "region"
	s3URLKey                 = "s3Url"
	publicURLKey             = "publicUrl"
	kmsKeyIDKey              = "kmsKeyId"
	s3ForcePathStyleKey      = "s3ForcePathStyle"
	bucketKey                = "bucket"
	signatureVersionKey      = "signatureVersion"
	credentialProfileKey     = "profile"
	serverSideEncryptionKey  = "serverSideEncryption"
	insecureSkipTLSVerifyKey = "insecureSkipTLSVerify"
	caCertKey                = "caCert"
)

func main() {

	//velero backup-location create cisco --bucket migration --prefix velero --provider aws --namespace openshift-migration --config "s3Url=https://<insert-your-urls>" --config "insecureSkipTLSVerify=true" --config "region=us-east-1" --config "s3ForcePathStyle=true"
	// populated ~/.aws/credentials with right default keys
	config := map[string]string{
		"regionKey":             "us-east-1",
		"s3Url":                 "https://<fill-in-your-values>",
		"s3ForcePathStyle":      "true",
		"profile":               "default",
		"insecureSkipTLSVerify": "true",
		"caCert":                "",
	}

	var (
		region = config[regionKey]
		s3URL  = config[s3URLKey]
		//publicURL                = config[publicURLKey]
		//kmsKeyID                 = config[kmsKeyIDKey]
		s3ForcePathStyleVal = config[s3ForcePathStyleKey]
		//signatureVersion         = config[signatureVersionKey]
		credentialProfile = config[credentialProfileKey]
		//serverSideEncryption     = config[serverSideEncryptionKey]
		insecureSkipTLSVerifyVal = config[insecureSkipTLSVerifyKey]

		// note that bucket is automatically added to the config map
		// by the server from the ObjectStorageProviderConfig so
		// doesn't need to be explicitly set by the user within
		// config.
		//bucket                = config[bucketKey]
		caCert                = config[caCertKey]
		s3ForcePathStyle      bool
		insecureSkipTLSVerify bool
		err                   error
	)

	if s3ForcePathStyleVal != "" {
		if s3ForcePathStyle, err = strconv.ParseBool(s3ForcePathStyleVal); err != nil {
			fmt.Errorf("error parsing bool: %#v", err)
			os.Exit(1)
		}
	}

	region = "us-east-1"

	serverConfig, err := newAWSConfig(s3URL, region, s3ForcePathStyle)
	if err != nil {
		fmt.Errorf("error getting aws config: %#v", err)
		os.Exit(1)
	}

	if insecureSkipTLSVerifyVal != "" {
		if insecureSkipTLSVerify, err = strconv.ParseBool(insecureSkipTLSVerifyVal); err != nil {
			fmt.Errorf("error parsing bool: %#v", err)
			os.Exit(1)
		}
	}

	if insecureSkipTLSVerify {
		defaultTransport := http.DefaultTransport.(*http.Transport)
		serverConfig.HTTPClient = &http.Client{
			// Copied from net/http
			Transport: &http.Transport{
				Proxy:                 defaultTransport.Proxy,
				DialContext:           defaultTransport.DialContext,
				MaxIdleConns:          defaultTransport.MaxIdleConns,
				IdleConnTimeout:       defaultTransport.IdleConnTimeout,
				TLSHandshakeTimeout:   defaultTransport.TLSHandshakeTimeout,
				ExpectContinueTimeout: defaultTransport.ExpectContinueTimeout,
				// Set insecureSkipVerify true
				TLSClientConfig: &tls.Config{
					InsecureSkipVerify: true,
				},
			},
		}
	}

	fmt.Println("here")

	sessionOptions := session.Options{Config: *serverConfig, Profile: credentialProfile}
	if len(caCert) > 0 {
		sessionOptions.CustomCABundle = strings.NewReader(caCert)
	}
	fmt.Println("here 2")
	serverSession, err := getSession(sessionOptions)
	if err != nil {
		fmt.Errorf("error getting server session: %#v", err)
		os.Exit(1)
	}

	fmt.Println("here 2")
	s3_client := s3.New(serverSession)
	s3_uploader := s3manager.NewUploader(serverSession)

	fmt.Printf("%#v %#v\n", s3_client, s3_uploader)

	bucket := "migration"
	prefix := "velero/backups/"
	delimiter := "/"
	//
	//reqParams := &s3.ListObjectsInput{
	//	Bucket:    &bucket,
	//	Prefix:    &prefix,
	//	Delimiter: &delimiter,
	//}
	//
	reqParams := &s3.ListObjectsV2Input{
		Bucket:    &bucket,
		Prefix:    &prefix,
		Delimiter: &delimiter,
	}

	//if publicURL != "" {
	//	publicConfig, err := newAWSConfig(publicURL, region, s3ForcePathStyle)
	//	if err != nil {
	//		fmt.Errorf("error getting public url aws config: %#v", err)
	//		os.Exit(1)
	//
	//	}
	//	sessionOptions := session.Options{Config: *publicConfig, Profile: credentialProfile}
	//	if len(caCert) > 0 {
	//		sessionOptions.CustomCABundle = strings.NewReader(caCert)
	//	}
	//	publicSession, err := getSession(sessionOptions)
	//	if err != nil {
	//		fmt.Errorf("error getting public url aws session: %#v", err)
	//		os.Exit(1)
	//	}
	//} else {
	//	o.preSignS3 = o.s3
	//}

	var ret []string
	tryV1 := false
	err = s3_client.ListObjectsV2Pages(reqParams, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
		for _, prefix := range page.CommonPrefixes {
			ret = append(ret, *prefix.Prefix)
		}
		if page.NextContinuationToken == nil && *page.IsTruncated {
			fmt.Println("this is not a valid v2 response")
			tryV1 = true
		}
		return !lastPage
	})
	if err != nil {
		fmt.Errorf("error running pager: %#v", err)
		os.Exit(1)
	}

	if tryV1 {
		// lets try all of the above again
		reqParams := &s3.ListObjectsInput{
			Bucket:    &bucket,
			Prefix:    &prefix,
			Delimiter: &delimiter,
		}

		ret = []string{}

		err = s3_client.ListObjectsPages(reqParams, func(page *s3.ListObjectsOutput, lastPage bool) bool {
			for _, prefix := range page.CommonPrefixes {
				ret = append(ret, *prefix.Prefix)
			}
			if page.NextMarker == nil && *page.IsTruncated {
				fmt.Println("this is not a valid v1 response")
			}
			return !lastPage
		})
		if err != nil {
			fmt.Errorf("error running pager: %#v", err)
			os.Exit(1)
		}
	}

	fmt.Println(len(ret))

	//reqParams2 := &s3.ListObjectsInput{
	//	Bucket:    &bucket,
	//	Prefix:    &prefix,
	//	Delimiter: &delimiter,
	//}
	//var ret2 []string
	//resp2, err := s3_client.ListObjects(reqParams2)
	//if err != nil {
	//	fmt.Errorf("error getting all objects: %#v", err)
	//	os.Exit(1)
	//}
	//for _, prefix := range resp2.CommonPrefixes {
	//	fmt.Printf("prefix: %s\n", *prefix.Prefix)
	//	ret2 = append(ret2, *prefix.Prefix)
	//}
	////startAfter := ret2[len(ret2)-1]
	//if *resp2.IsTruncated {
	//	reqParams3 := &s3.ListObjectsInput{
	//		Bucket:    &bucket,
	//		Prefix:    &prefix,
	//		Delimiter: &delimiter,
	//		Marker:    resp2.NextMarker,
	//	}
	//	resp3, err := s3_client.ListObjects(reqParams3)
	//	if err != nil {
	//		fmt.Errorf("error getting resp3: %#v", err)
	//		os.Exit(1)
	//	}
	//	for _, prefix := range resp3.CommonPrefixes {
	//		fmt.Printf("prefix: %s\n", *prefix.Prefix)
	//		ret2 = append(ret2, *prefix.Prefix)
	//	}
	//}
	//fmt.Println(len(ret2))
}

// takes AWS session options to create a new session
func getSession(options session.Options) (*session.Session, error) {
	sess, err := session.NewSessionWithOptions(options)
	if err != nil {
		return nil, errors.WithStack(err)
	}

	if _, err := sess.Config.Credentials.Get(); err != nil {
		return nil, errors.WithStack(err)
	}
	return sess, nil
}

func newAWSConfig(url, region string, forcePathStyle bool) (*aws.Config, error) {
	awsConfig := aws.NewConfig().
		WithRegion(region).
		WithS3ForcePathStyle(forcePathStyle)

	if url != "" {
		if !IsValidS3URLScheme(url) {
			return nil, errors.Errorf("Invalid s3 url %s, URL must be valid according to https://golang.org/pkg/net/url/#Parse and start with http:// or https://", url)
		}

		awsConfig = awsConfig.WithEndpointResolver(
			endpoints.ResolverFunc(func(service, region string, optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) {
				if service == endpoints.S3ServiceID {
					return endpoints.ResolvedEndpoint{
						URL: url,
					}, nil
				}

				return endpoints.DefaultResolver().EndpointFor(service, region, optFns...)
			}),
		)
	}

	return awsConfig, nil
}

// IsValidS3URLScheme returns true if the scheme is http:// or https://
// and the url parses correctly, otherwise, return false
func IsValidS3URLScheme(s3URL string) bool {
	u, err := url.Parse(s3URL)
	if err != nil {
		return false
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		return false
	}
	return true
}