Argo CD Application Controller Custom Sharding

README.md

Argo CD Application Controller Custom Sharding

This short Go program uses a Fist-fit-decreasing Bin Packing algorithm to assign Kubernetes clusters to different Argo CD Application Controller shard based on their resources count. It bases its calculations on the output of the argocd-util cluster stats command. The output can be used to update the undocumented shard key of the cluster secrets.

Example usage

argocd-util cluster stats --namespace=argocd | go run reshard.go

Example output

Fields are server shard resourcesCount (clusters are on STDOUT, extra stats on STDERR):

https://kube-api-server.cluster1.org 1 8474
https://kube-api-server.cluster2.org 2 6402
https://kube-api-server.cluster3.org 3 6313
https://kube-api-server.cluster4.org 4 5479
https://kube-api-server.cluster5.org 5 4059
https://kube-api-server.cluster6.org 5 4028
https://kube-api-server.cluster7.org 6 3188
https://kube-api-server.cluster8.org 6 3131
https://kube-api-server.cluster9.org 7 2786
https://kube-api-server.cluster10.org 7 2630
https://kubernetes.default.svc 7 2594
https://kube-api-server.cluster12.org 8 2330
https://kube-api-server.cluster13.org 8 2302
https://kube-api-server.cluster14.org 8 2178
https://kube-api-server.cluster15.org 9 2093
https://kube-api-server.cluster16.org 9 1840
https://kube-api-server.cluster17.org 9 1722
https://kube-api-server.cluster18.org 9 1649
https://kube-api-server.cluster19.org 10 1577
https://kube-api-server.cluster20.org 10 1482
https://kube-api-server.cluster21.org 10 1434
https://kube-api-server.cluster22.org 10 1413
https://kube-api-server.cluster23.org 10 1300
https://kube-api-server.cluster24.org 11 1285
https://kube-api-server.cluster25.org 11 968
Total resources count: 72657
Maximum resources count: 8474
Number of controllers: 12
Average controller usage: 0.71450908661789

This can easily be parsed with a tool like awk, and used to automate the update of your Kubernetes manifests or patching in Kubernetes directly.

Notes

• This code is meant as a proof of concept and is voluntarily minimalist. It does not do a lot of error handling and input checking, so I would not be surprised if it breaks for you.
• argocd-util command has been moved to argocd admin in Argo CD 2.1+.
• Some caveats:
  • Resources count correlation with memory usage is not 100% accurate, it may be depend on a combination of the number of applications and resources or some other parameters.
  • The First-fit algorithm may not be the best, its allocations can be uneven, the biggest cluster will use "100%" of its controller, but the small might take a extra one. for itself too (while it could be paired with a medium size cluster instead).
• Please see argoproj/argo-cd#6125 for more context.

reshard.go

package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

type Cluster struct {
	Server         string
	ResourcesCount int
	Shard          int
}

func sortClustersByResourcesCountDesc(clusters []*Cluster) {
	sort.SliceStable(clusters, func(i, j int) bool {
		return clusters[i].ResourcesCount > clusters[j].ResourcesCount
	})
}

func firstFit(clusters []*Cluster, capacity int) (int, int) {
	controllerID := 0
	currentControllerResources := 0
	totalResourcesCount := 0

	for _, c := range clusters {
		if c.ResourcesCount > currentControllerResources {
			controllerID++

			currentControllerResources = capacity
		}

		c.Shard = controllerID
		currentControllerResources -= c.ResourcesCount
		totalResourcesCount += c.ResourcesCount
	}

	return controllerID + 1, totalResourcesCount
}

func parseClusterStats(r io.Reader) []*Cluster {
	var (
		serverIndex         = -1
		resourcesCountIndex = -1
		resourcesCount      int
		line                []string
		clusters            []*Cluster
	)

	scanner := bufio.NewScanner(r)
	singleSpaceRe := regexp.MustCompile(`(\w)\s(\w)`)

	scanner.Scan()
	// Rename field names with spaces to use underscores instead,
	// so strings.Fields() does not split them
	// e.g. `RESOURCES COUNT` -> `RESOURCES_COUNT`
	head := singleSpaceRe.ReplaceAllString(scanner.Text(), `${1}_${2}`)
	headers := strings.Fields(head)

	for i, h := range headers {
		if h == "SERVER" {
			serverIndex = i
		} else if h == "RESOURCES_COUNT" {
			resourcesCountIndex = i
		}
	}

	for scanner.Scan() {
		line = strings.Fields(scanner.Text())
		resourcesCount, _ = strconv.Atoi(line[resourcesCountIndex])

		clusters = append(clusters, &Cluster{
			Server:         line[serverIndex],
			ResourcesCount: resourcesCount,
			Shard:          0,
		})
	}

	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "Error while reading stdin:", err)
	}

	return clusters
}

func main() {
	clusters := parseClusterStats(os.Stdin)
	sortClustersByResourcesCountDesc(clusters)
	maxResources := clusters[0].ResourcesCount
	numControllers, totalResources := firstFit(clusters, maxResources)
	avgUsage := float64(totalResources) / float64(maxResources * numControllers)

	for _, c := range clusters {
		fmt.Fprintln(os.Stdout, c.Server, c.Shard, c.ResourcesCount)
	}

	fmt.Fprintln(os.Stderr, "Total resources count:", totalResources)
	fmt.Fprintln(os.Stderr, "Maximum resources count:", maxResources)
	fmt.Fprintln(os.Stderr, "Number of controllers:", numControllers)
	fmt.Fprintln(os.Stderr, "Average controller usage:", avgUsage)
}