Connect to Google Kubernetes with GCP credentials and pure Golang

gcp-kube.go

package main

import (
	"context"
	"encoding/base64"
	"flag"
	"fmt"
	"log"

	container "google.golang.org/api/container/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/clientcmd/api"
	_ "k8s.io/cloud-provider-gcp/pkg/clientauthplugin/gcp" // register GCP auth provider
)

var fProjectId = flag.String("projectId", "", "specify a project id to examine")

func main() {
	flag.Parse()
	if *fProjectId == "" {
		log.Fatal("must specific -projectId")
	}

	if err := run(context.Background(), *fProjectId); err != nil {
		log.Fatal(err)
	}
}

func run(ctx context.Context, projectId string) error {
	kubeConfig, err := getK8sClusterConfigs(ctx, projectId)
	if err != nil {
		return err
	}

	// Just list all the namespaces found in the project to test the API.
	for clusterName := range kubeConfig.Clusters {
		cfg, err := clientcmd.NewNonInteractiveClientConfig(*kubeConfig, clusterName, &clientcmd.ConfigOverrides{CurrentContext: clusterName}, nil).ClientConfig()
		if err != nil {
			return fmt.Errorf("failed to create Kubernetes configuration cluster=%s: %w", clusterName, err)
		}

		k8s, err := kubernetes.NewForConfig(cfg)
		if err != nil {
			return fmt.Errorf("failed to create Kubernetes client cluster=%s: %w", clusterName, err)
		}

		ns, err := k8s.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
		if err != nil {
			return fmt.Errorf("failed to list namespaces cluster=%s: %w", clusterName, err)
		}

		log.Printf("Namespaces found in cluster=%s", clusterName)

		for _, item := range ns.Items {
			log.Println(item.Name)
		}
	}

	return nil
}

func getK8sClusterConfigs(ctx context.Context, projectId string) (*api.Config, error) {
	svc, err := container.NewService(ctx)
	if err != nil {
		return nil, fmt.Errorf("container.NewService: %w", err)
	}

	// Basic config structure
	ret := api.Config{
		APIVersion: "v1",
		Kind:       "Config",
		Clusters:   map[string]*api.Cluster{},  // Clusters is a map of referencable names to cluster configs
		AuthInfos:  map[string]*api.AuthInfo{}, // AuthInfos is a map of referencable names to user configs
		Contexts:   map[string]*api.Context{},  // Contexts is a map of referencable names to context configs
	}

	// Ask Google for a list of all kube clusters in the given project.
	resp, err := svc.Projects.Zones.Clusters.List(projectId, "-").Context(ctx).Do()
	if err != nil {
		return nil, fmt.Errorf("clusters list project=%s: %w", projectId, err)
	}

	for _, f := range resp.Clusters {
		name := fmt.Sprintf("gke_%s_%s_%s", projectId, f.Zone, f.Name)
		cert, err := base64.StdEncoding.DecodeString(f.MasterAuth.ClusterCaCertificate)
		if err != nil {
			return nil, fmt.Errorf("invalid certificate cluster=%s cert=%s: %w", name, f.MasterAuth.ClusterCaCertificate, err)
		}
		// example: gke_my-project_us-central1-b_cluster-1 => https://XX.XX.XX.XX
		ret.Clusters[name] = &api.Cluster{
			CertificateAuthorityData: cert,
			Server:                   "https://" + f.Endpoint,
		}
		// Just reuse the context name as an auth name.
		ret.Contexts[name] = &api.Context{
			Cluster:  name,
			AuthInfo: name,
		}
		// GCP specific configation; use cloud platform scope.
		ret.AuthInfos[name] = &api.AuthInfo{
			AuthProvider: &api.AuthProviderConfig{
				Name: "gcp",
				Config: map[string]string{
					"scopes": "https://www.googleapis.com/auth/cloud-platform",
				},
			},
		}
	}

	return &ret, nil
}

go.mod

module github.com/dragonsinth/gcp-kube

go 1.21.6

require (
	google.golang.org/api v0.63.0
	k8s.io/apimachinery v0.26.0
	k8s.io/client-go v0.26.0
	k8s.io/cloud-provider-gcp v0.0.0-20230119221216-bb1acae5826d
)

require (
	cloud.google.com/go v0.99.0 // indirect
	github.com/davecgh/go-spew v1.1.1 // indirect
	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
	github.com/go-logr/logr v1.2.3 // indirect
	github.com/go-openapi/jsonpointer v0.19.5 // indirect
	github.com/go-openapi/jsonreference v0.20.0 // indirect
	github.com/go-openapi/swag v0.19.14 // indirect
	github.com/gogo/protobuf v1.3.2 // indirect
	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
	github.com/golang/protobuf v1.5.2 // indirect
	github.com/google/gnostic v0.5.7-v3refs // indirect
	github.com/google/go-cmp v0.5.9 // indirect
	github.com/google/gofuzz v1.1.0 // indirect
	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
	github.com/imdario/mergo v0.3.11 // indirect
	github.com/josharian/intern v1.0.0 // indirect
	github.com/json-iterator/go v1.1.12 // indirect
	github.com/mailru/easyjson v0.7.6 // indirect
	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
	github.com/modern-go/reflect2 v1.0.2 // indirect
	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
	github.com/spf13/pflag v1.0.5 // indirect
	go.opencensus.io v0.23.0 // indirect
	golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect
	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
	golang.org/x/sys v0.3.0 // indirect
	golang.org/x/term v0.3.0 // indirect
	golang.org/x/text v0.5.0 // indirect
	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
	google.golang.org/appengine v1.6.7 // indirect
	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
	google.golang.org/grpc v1.49.0 // indirect
	google.golang.org/protobuf v1.28.1 // indirect
	gopkg.in/inf.v0 v0.9.1 // indirect
	gopkg.in/yaml.v2 v2.4.0 // indirect
	gopkg.in/yaml.v3 v3.0.1 // indirect
	k8s.io/api v0.26.0 // indirect
	k8s.io/klog/v2 v2.80.1 // indirect
	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
	sigs.k8s.io/yaml v1.3.0 // indirect
)

Just wanted to say thank you and this helped answer a question I had. The other thing I needed to do was generate a token using https://pkg.go.dev/google.golang.org/api/transport#Creds and place it inside of api.AuthInfo{}

@benpoliquin glad it was helpful!

Did you get here from my blog post?

@benpoliquin glad it was helpful!

Did you get here from my blog post?

That I did. Thank you for the write-up!

@benpoliquin glad it was helpful!

Did you get here from my blog post?

Oh man i cannot thank you enough, your post single handedly saved me.
However, for some reason finding your post while googling took some time. Just when I about to gave up i tried searching "k8s.io/client-go/plugin/pkg/client/auth/gcp" and your post was no 5 in the search list.

if your title was "authenticating to gke without kubeconfig" or similar, should be picked up by google search.

it was a great post , i cannot appreciate enough and i believe we can use same approach for AKS or EKS ?

@shakilbd009 oh man, sorry it was hard to find, but thank you very much for the suggestion. I've updated the blog post to add a subtitle, so who knows maybe Google will find it more easily under those search terms in the future!

EDIT: looks like we're now on page 1 for "authenticate to gke without kubeconfig". Thanks so much for the idea!

I don't know about authenticating to other Kube providers-- I only have access to GKE myself. But I would imagine something similar should work?

Just wanted to say thank you and this helped answer a question I had. The other thing I needed to do was generate a token using https://pkg.go.dev/google.golang.org/api/transport#Creds and place it inside of api.AuthInfo{}

@benpoliquin can you paste some snippet of how to do this?

@zreigz

	creds, err := transport.Creds(context.Background(), option.WithCredentialsFile("path/to/keyfile.json"), option.WithScopes("https://www.googleapis.com/auth/cloud-platform"))
	if err != nil {
		return fmt.Errorf("generating gcp creds: %w", err)
	}
	tkn, err := creds.TokenSource.Token()
	if err != nil {
		return fmt.Errorf("generating token: %w", err)
	}
	ret.AuthInfos[name] = &k8sapi.AuthInfo{
		Token: tkn.AccessToken,
		AuthProvider: &k8sapi.AuthProviderConfig{
			Name: "gcp",
			Config: map[string]string{
				"scopes": "https://www.googleapis.com/auth/cloud-platform",
			},
		},
	}

@nstogner thanks a lot. I used this method for the token:

creds, err := google.CredentialsFromJSON(ctx, b, container.CloudPlatformScope)

I am getting following error while executing

2023/01/23 11:47:49 failed to create Kubernetes client cluster=<cluster_name>: The gcp auth plugin has been removed.
Please use the "gke-gcloud-auth-plugin" kubectl/client-go credential plugin instead.
See https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke for further details
exit status 1

I'll have to re-test, there's a new this works in 1.2.26, with respect to the CLI. I'm not sure what impact this has on pure Go. I'll dig around and see what I find.

Okay, I have an updated solution that should work with newer libs.

1. In your go.mod, replace k8s.io/cloud-provider-gcp/providers => k8s.io/cloud-provider-gcp/providers v0.25.5

2. go get k8s.io/cloud-provider-gcp/pkg/clientauthplugin/gcp@bb1acae5826dc877953d4854faf414e860db2efa

3. Change the import: _ "k8s.io/cloud-provider-gcp/pkg/clientauthplugin/gcp" // register GCP auth provider plugin.

The code moved here (but note the library is unsupported): https://github.com/kubernetes/cloud-provider-gcp/tree/bb1acae5826dc877953d4854faf414e860db2efa/pkg/clientauthplugin

Hey @dragonsinth it's working for me. But when i try to import same with helm packages it is upgrading the kubernetes dependency

i am trying to import the helm dependency :

"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/cli"

I want to fetch all the installed helm releases from running cluster.
But when i do

go get helm.sh/helm/v3/pkg/action
go get helm.sh/helm/v3/pkg/cli

In go.mod file it is upgrading the version of k8s

k8s.io/client-go v0.25.5 - > v0.26.0

Can you please help me with this issue.

@eben-rockx see this comment, this worked for me at least
https://gist.github.com/dragonsinth/aea365732b60da3adc928dc18fff56ed?permalink_comment_id=4446169#gistcomment-4446169

Client dependencies being upgraded forcefully after installing the helm modules

go get helm.sh/helm/v3/pkg/action
go get helm.sh/helm/v3/pkg/cli

k8s.io/client-go v0.25.5 - > v0.26.0

To simulate the issue :
Can you please try to import the helm module, after that you will see it is not working.

@ucguy4u I'm on k8s.io/client-go v0.26.0 and it works fine for me; v0.26.1 also works fine

I'm on k8s.io/client-go v0.30.0 which is latest as of today and this is still working as suggested. Thank you so much. This still helps me implement my stuff in pure go so far, last option would be to use the CLI if it breaks someday.

👍 glad folks are still finding this useful. I went ahead and made an update for go 1.21.6 and included a working go.mod file here; I think this should just work out of the box in go 1.21 now. (I didn't bother including a go.sum -- seemed spammy? But I could attach my go.sum as well if that's useful for folks.)

Even it's working for 1.22.2 which is latest as of today. Glad that you added go.mod. Thanks.

Interestingly, when I updated today I found I no longer needed the replace in the go mod... I wonder if that means this could now be transformed into a tiny shared lib.

Yes, you are right, I didn't mention it earlier, the first step in the comment is no longer needed.

Is there a more updated version that is not using clientauthplugin?

Per docs:

Client auth library is a copy of the gcp plugin library that existed in Client-Go until k8s 1.25. Going ahead, this library is not supported and is expected to be replaced by GKE-GCLOUD-AUTH-PLUGIN. The code here is kept as a reference, in case anyone needs to refer to it.

https://github.com/kubernetes/cloud-provider-gcp/tree/master/pkg/clientauthplugin

@Tang8330 feel free to hack on one, I'm stiil using the old package. According to the new docs:

You will need to install the gke-gcloud-auth-plugin binary on all systems where kubectl or Kubernetes custom clients are used.

I wanted a pure Go solution, whereas it looks like the new stuff requires an installed binary.

@dragonsinth I know :'(. I have it working by installing the binary, but it feels dirty.