Case 48810248: Unable to authenticate to Artifact Registry using Workload Identity

0-gmp-ce-depl.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: flask-example
spec:
  selector:
    matchLabels:
      app: flask-example
  template:
    metadata:
      labels:
        app: flask-example
    spec:
      serviceAccountName: artifact-registry-sa
      containers:
        - name: flask-example
          image: us-docker.pkg.dev/clov-prod-gke-5afc/clovis/custom-collector-demo:0.1.0
          ports:
            - containerPort: 8080

1-default-flask-example-7ccb5cdf89-g244j

Name:             flask-example-7ccb5cdf89-g244j
Namespace:        default
Priority:         0
Service Account:  artifact-registry-sa
Node:             gke-clov-use1-prod-hub--use1-standard-065b0f75-mf7p/10.100.0.15
Start Time:       Thu, 04 Jan 2024 15:12:45 -0600
Labels:           app=flask-example
                  pod-template-hash=7ccb5cdf89
Annotations:      cni.projectcalico.org/containerID: 2190c824da76458ae1365508f439d36d2a848988c1aebb3812ec9391322ffd3c
                  cni.projectcalico.org/podIP: 10.3.2.17/32
                  cni.projectcalico.org/podIPs: 10.3.2.17/32
Status:           Pending
IP:               10.3.2.17
IPs:
  IP:           10.3.2.17
Controlled By:  ReplicaSet/flask-example-7ccb5cdf89
Containers:
  flask-example:
    Container ID:   
    Image:          us-docker.pkg.dev/clov-prod-gke-5afc/clovis/custom-collector-demo:0.1.0
    Image ID:       
    Port:           8080/TCP
    Host Port:      0/TCP
    State:          Waiting
      Reason:       ErrImagePull
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-r2p82 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  kube-api-access-r2p82:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason     Age                  From               Message
  ----     ------     ----                 ----               -------
  Normal   Scheduled  11m                  default-scheduler  Successfully assigned default/flask-example-7ccb5cdf89-g244j to gke-clov-use1-prod-hub--use1-standard-065b0f75-mf7p
  Normal   Pulling    9m52s (x4 over 11m)  kubelet            Pulling image "us-docker.pkg.dev/clov-prod-gke-5afc/clovis/custom-collector-demo:0.1.0"
  Warning  Failed     9m52s (x4 over 11m)  kubelet            Failed to pull image "us-docker.pkg.dev/clov-prod-gke-5afc/clovis/custom-collector-demo:0.1.0": rpc error: code = Unknown desc = failed to pull and unpack image "us-docker.pkg.dev/clov-prod-gke-5afc/clovis/custom-collector-demo:0.1.0": failed to resolve reference "us-docker.pkg.dev/clov-prod-gke-5afc/clovis/custom-collector-demo:0.1.0": failed to authorize: failed to fetch anonymous token: unexpected status from GET request to https://us-docker.pkg.dev/v2/token?scope=repository%3Aclov-prod-gke-5afc%2Fclovis%2Fcustom-collector-demo%3Apull&service=us-docker.pkg.dev: 403 Forbidden
  Warning  Failed     9m52s (x4 over 11m)  kubelet            Error: ErrImagePull
  Warning  Failed     9m40s (x6 over 11m)  kubelet            Error: ImagePullBackOff
  Normal   BackOff    66s (x43 over 11m)   kubelet            Back-off pulling image "us-docker.pkg.dev/clov-prod-gke-5afc/clovis/custom-collector-demo:0.1.0"

2-default-artifact-registry-sa.yaml

apiVersion: v1
automountServiceAccountToken: true
kind: ServiceAccount
metadata:
  annotations:
    iam.gke.io/gcp-service-account: artifact-registry-sa@clov-prod-gke-5afc.iam.gserviceaccount.com
  creationTimestamp: "2023-12-30T21:56:50Z"
  name: artifact-registry-sa
  namespace: default
  resourceVersion: "21831885"
  uid: 8ee88f45-500f-4f21-b814-db7e78955c65