OLM - Checking APIService Status

actionable.go

// apiServiceResourceErrorsActionable returns true if OLM can do something about any one
// of the apiService errors in errs; otherwise returns false
//
// This method can be used to determine if a CSV in a failed state due to APIService
// issues can resolve them by reinstalling
func (a *Operator) apiServiceResourceErrorsActionable(errs []error) bool {
  for _, err := range errs {
    switch e := err.(type){
      case UnadoptableError:
        return false 
    }
  }
  
  return true
}

type UnadoptableError struct {
  resourceNamespace string
  resourceName  string
}

func (err UnadoptableError) Error() string {
  if err.resourceNamespace == "" {
    return fmt.Sprintf("%s is unadoptable", err.resourceName)
  }
  return fmt.Sprintf("%s/%s is unadoptable", err.resourceNamespace, err.resourceName)
}

func NewUnadoptableError(resourceNamespace, resourceName string) UnadoptableError {
  return UnadoptableError{resourceNamespace, resourceName}
}

// checkAPIServiceResources checks if all expected generated resources for the given APIService exist
func (a *Operator) checkAPIServiceResources(csv, replacement *v1alpha1.ClusterServiceVersion, hashFunc certs.PEMHash) []error {
  errs := []error{}
  owners := []ownerutil.Owner{csv}
  if replacement != nil {
    owners = append(owners, replacement)
  }
	ruleChecker := install.NewCSVRuleChecker(a.lister.RbacV1().RoleLister(), a.lister.RbacV1().RoleBindingLister(), a.lister.RbacV1().ClusterRoleLister(), a.lister.RbacV1().ClusterRoleBindingLister(), csv)
	
  for _, desc := range csv.GetOwnedAPIServiceDescriptions() {
		apiServiceName := fmt.Sprintf("%s.%s", desc.Version, desc.Group)
		logger := log.WithFields(log.Fields{
			"csv":        csv.GetName(),
			"namespace":  csv.GetNamespace(),
			"apiservice": apiServiceName,
		})

		apiService, err := a.lister.APIRegistrationV1().APIServiceLister().Get(apiServiceName)
		if err != nil {
			logger.Warnf("could not retrieve generated APIService")
			errs := append(errs, err)
      continue
		}
    
    // Check if the APIService is adoptable
    if !ownerutil.OwnersIntersect(owners, apiService.GetOwnerReferences()) {
      err := NewUnadpotableError("", apiServiceName)
      logger.WithError(err).Warn("found unadoptable apiservice")
      errs = append(err, errs)
      continue
    }
    
    serviceName := APIServiceNameToServiceName(apiServiceName)
		service, err := a.lister.CoreV1().ServiceLister().Services(csv.GetNamespace()).Get(serviceName)
		if err != nil {
			logger.WithField("service", serviceName).Warnf("could not retrieve generated Service")
      errs := append(errs, err)
      continue
		}

		// Check if the APIService points to the correct service
		if apiService.Spec.Service.Name != serviceName || apiService.Spec.Service.Namespace != csv.GetNamespace() {
			logger.WithFields(log.Fields{"service": apiService.Spec.Service.Name, "serviceNamespace": apiService.Spec.Service.Namespace}).Warnf("APIService service reference mismatch")
			errs := append(errs, fmt.Errorf("APIService service reference mismatch"))
      continue
		}

		// Check if CA is Active
		caBundle := apiService.Spec.CABundle
		ca, err := certs.PEMToCert(caBundle)
		if err != nil {
			logger.Warnf("could not convert APIService CA bundle to x509 cert")
			errs := append(errs, err)
      continue
		}
		if !certs.Active(ca) {
			logger.Warnf("CA cert not active")
      errs := append(errs, fmt.Errorf("CA cert not active"))
      continue
		}

		// Check if serving cert is active
		secretName := apiServiceName + "-cert"
		secret, err := a.lister.CoreV1().SecretLister().Secrets(csv.GetNamespace()).Get(secretName)
		if err != nil {
			logger.WithField("secret", secretName).Warnf("could not retrieve generated Secret")
      errs := append(errs, err)
      continue
		}
		cert, err := certs.PEMToCert(secret.Data["tls.crt"])
		if err != nil {
			logger.Warnf("could not convert serving cert to x509 cert")
      errs := append(errs, err)
      continue
		}
		if !certs.Active(cert) {
			logger.Warnf("serving cert not active")
      errs := append(errs, fmt.Errorf("serving cert not active"))
      continue
		}

		// Check if CA hash matches expected
		caHash := hashFunc(caBundle)
		if hash, ok := secret.GetAnnotations()[OLMCAHashAnnotationKey]; !ok || hash != caHash {
			logger.WithField("secret", secretName).Warnf("secret CA cert hash does not match expected")
      errs := append(errs, fmt.Errorf("secret %s CA cert hash does not match expected", secretName))
      continue
		}

		// Check if serving cert is trusted by the CA
		hosts := []string{
			fmt.Sprintf("%s.%s", service.GetName(), csv.GetNamespace()),
			fmt.Sprintf("%s.%s.svc", service.GetName(), csv.GetNamespace()),
		}
		for _, host := range hosts {
			if err := certs.VerifyCert(ca, cert, host); err != nil {
        errs := append(errs, fmt.Errorf("could not verify cert: %s", err.Error()))
        continue
			}
		}

		// Ensure the existing Deployment has a matching CA hash annotation
		deployment, err := a.lister.AppsV1().DeploymentLister().Deployments(csv.GetNamespace()).Get(desc.DeploymentName)
		if k8serrors.IsNotFound(err) || err != nil {
			logger.WithField("deployment", desc.DeploymentName).Warnf("expected Deployment could not be retrieved")
      errs := append(errs, err)
      continue
		}
		if hash, ok := deployment.Spec.Template.GetAnnotations()[OLMCAHashAnnotationKey]; !ok || hash != caHash {
			logger.WithField("deployment", desc.DeploymentName).Warnf("Deployment CA cert hash does not match expected")
			errs := append(errs, fmt.Errorf("Deployment %s CA cert hash does not match expected", desc.DeploymentName))
      continue
		}

		// Ensure the Deployment's ServiceAccount exists
		serviceAccountName := deployment.Spec.Template.Spec.ServiceAccountName
		if serviceAccountName == "" {
			serviceAccountName = "default"
		}
		serviceAccount, err := a.lister.CoreV1().ServiceAccountLister().ServiceAccounts(deployment.GetNamespace()).Get(serviceAccountName)
		if err != nil {
			logger.WithField("serviceaccount", serviceAccountName).Warnf("could not retrieve ServiceAccount")
      errs := append(errs, err)
      continue
		}

		// Ensure RBAC permissions for the APIService are correct
		rulesMap := map[string][]rbacv1.PolicyRule{
			// Serving cert Secret Rule
			csv.GetNamespace(): {
				{
					Verbs:         []string{"get"},
					APIGroups:     []string{""},
					Resources:     []string{"secrets"},
					ResourceNames: []string{secret.GetName()},
				},
			},
			"kube-system":       {},
			metav1.NamespaceAll: {},
		}

		// extension-apiserver-authentication-reader
		authReaderRole, err := a.lister.RbacV1().RoleLister().Roles("kube-system").Get("extension-apiserver-authentication-reader")
		if err != nil {
			logger.Warnf("could not retrieve Role extension-apiserver-authentication-reader")
			errs := append(errs, err)
      continue
		}
		rulesMap["kube-system"] = append(rulesMap["kube-system"], authReaderRole.Rules...)

		// system:auth-delegator
		authDelegatorClusterRole, err := a.lister.RbacV1().ClusterRoleLister().Get("system:auth-delegator")
		if err != nil {
			logger.Warnf("could not retrieve ClusterRole system:auth-delegator")
			errs := append(errs, err)
      continue
		}
		rulesMap[metav1.NamespaceAll] = append(rulesMap[metav1.NamespaceAll], authDelegatorClusterRole.Rules...)

		for namespace, rules := range rulesMap {
			for _, rule := range rules {
				satisfied, err := ruleChecker.RuleSatisfied(serviceAccount, namespace, rule)
				if err != nil {
					logger.WithField("rule", fmt.Sprintf("%+v", rule)).Warnf("error checking Rule")
					errs := append(errs, err)
          continue
				}
				if !satisfied {
					logger.WithField("rule", fmt.Sprintf("%+v", rule)).Warnf("Rule not satisfied")
          errs := append(errs, fmt.Errorf("Rule %+v not satisfied", rule)
			 continue
				}
			}
		}
	}

	return errs
}