Skip to content

Instantly share code, notes, and snippets.

@esys

esys/autoscaler-eks.ts

Last active Feb 12, 2021
Embed
What would you like to do?
eks cdk install cluster autoscaler
import * as cdk from "@aws-cdk/core";
import * as eks from "@aws-cdk/aws-eks";
import * as ec2 from "@aws-cdk/aws-ec2";
import * as iam from "@aws-cdk/aws-iam";
import { CfnJson } from "@aws-cdk/core";
export class MyCluster extends cdk.Construct {
constructor(scope: cdk.Construct, id: string) {
super(scope, id);
const vpc = new ec2.Vpc(this, "eks-vpc");
const cluster = new eks.Cluster(this, "Cluster", {
vpc: vpc,
defaultCapacity: 0, // we want to manage capacity ourselves
version: eks.KubernetesVersion.V1_17,
});
// cluster.addCapacity("spot", {
// spotPrice: "0.0125",
// instanceType: new ec2.InstanceType("t3.medium"),
// maxCapacity: 2,
// });
const ng = cluster.addNodegroup("nodegroup", {
instanceType: new ec2.InstanceType("t3.medium"),
minSize: 1,
maxSize: 3,
remoteAccess: { sshKeyName: "eksworkshop" },
});
this.enableAutoscaling(cluster, ng);
}
enableAutoscaling(cluster: eks.Cluster, ng: eks.Nodegroup, version: string = "v1.17.3") {
const autoscalerStmt = new iam.PolicyStatement();
autoscalerStmt.addResources("*");
autoscalerStmt.addActions(
"autoscaling:DescribeAutoScalingGroups",
"autoscaling:DescribeAutoScalingInstances",
"autoscaling:DescribeLaunchConfigurations",
"autoscaling:DescribeTags",
"autoscaling:SetDesiredCapacity",
"autoscaling:TerminateInstanceInAutoScalingGroup",
"ec2:DescribeLaunchTemplateVersions"
);
const autoscalerPolicy = new iam.Policy(this, "cluster-autoscaler-policy", {
policyName: "ClusterAutoscalerPolicy",
statements: [autoscalerStmt],
});
autoscalerPolicy.attachToRole(ng.role);
const clusterName = new CfnJson(this, "clusterName", {
value: cluster.clusterName,
});
cdk.Tag.add(ng, `k8s.io/cluster-autoscaler/${clusterName}`, "owned", { applyToLaunchedInstances: true });
cdk.Tag.add(ng, "k8s.io/cluster-autoscaler/enabled", "true", { applyToLaunchedInstances: true });
new eks.KubernetesManifest(this, "cluster-autoscaler", {
cluster,
manifest: [
{
apiVersion: "v1",
kind: "ServiceAccount",
metadata: {
name: "cluster-autoscaler",
namespace: "kube-system",
labels: {
"k8s-addon": "cluster-autoscaler.addons.k8s.io",
"k8s-app": "cluster-autoscaler",
},
},
},
{
apiVersion: "rbac.authorization.k8s.io/v1",
kind: "ClusterRole",
metadata: {
name: "cluster-autoscaler",
namespace: "kube-system",
labels: {
"k8s-addon": "cluster-autoscaler.addons.k8s.io",
"k8s-app": "cluster-autoscaler",
},
},
rules: [
{
apiGroups: [""],
resources: ["events", "endpoints"],
verbs: ["create", "patch"],
},
{
apiGroups: [""],
resources: ["pods/eviction"],
verbs: ["create"],
},
{
apiGroups: [""],
resources: ["pods/status"],
verbs: ["update"],
},
{
apiGroups: [""],
resources: ["endpoints"],
resourceNames: ["cluster-autoscaler"],
verbs: ["get", "update"],
},
{
apiGroups: ["coordination.k8s.io"],
resources: ["leases"],
verbs: ["watch", "list", "get", "patch", "create", "update"],
},
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["watch", "list", "get", "update"],
},
{
apiGroups: [""],
resources: ["pods", "services", "replicationcontrollers", "persistentvolumeclaims", "persistentvolumes"],
verbs: ["watch", "list", "get"],
},
{
apiGroups: ["extensions"],
resources: ["replicasets", "daemonsets"],
verbs: ["watch", "list", "get"],
},
{
apiGroups: ["policy"],
resources: ["poddisruptionbudgets"],
verbs: ["watch", "list"],
},
{
apiGroups: ["apps"],
resources: ["statefulsets", "replicasets", "daemonsets"],
verbs: ["watch", "list", "get"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["storageclasses", "csinodes"],
verbs: ["watch", "list", "get"],
},
{
apiGroups: ["batch", "extensions"],
resources: ["jobs"],
verbs: ["get", "list", "watch", "patch"],
},
],
},
{
apiVersion: "rbac.authorization.k8s.io/v1",
kind: "Role",
metadata: {
name: "cluster-autoscaler",
namespace: "kube-system",
labels: {
"k8s-addon": "cluster-autoscaler.addons.k8s.io",
"k8s-app": "cluster-autoscaler",
},
},
rules: [
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["create", "list", "watch"],
},
{
apiGroups: [""],
resources: ["configmaps"],
resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"],
verbs: ["delete", "get", "update", "watch"],
},
],
},
{
apiVersion: "rbac.authorization.k8s.io/v1",
kind: "ClusterRoleBinding",
metadata: {
name: "cluster-autoscaler",
namespace: "kube-system",
labels: {
"k8s-addon": "cluster-autoscaler.addons.k8s.io",
"k8s-app": "cluster-autoscaler",
},
},
roleRef: {
apiGroup: "rbac.authorization.k8s.io",
kind: "ClusterRole",
name: "cluster-autoscaler",
},
subjects: [
{
kind: "ServiceAccount",
name: "cluster-autoscaler",
namespace: "kube-system",
},
],
},
{
apiVersion: "rbac.authorization.k8s.io/v1",
kind: "RoleBinding",
metadata: {
name: "cluster-autoscaler",
namespace: "kube-system",
labels: {
"k8s-addon": "cluster-autoscaler.addons.k8s.io",
"k8s-app": "cluster-autoscaler",
},
},
roleRef: {
apiGroup: "rbac.authorization.k8s.io",
kind: "Role",
name: "cluster-autoscaler",
},
subjects: [
{
kind: "ServiceAccount",
name: "cluster-autoscaler",
namespace: "kube-system",
},
],
},
{
apiVersion: "apps/v1",
kind: "Deployment",
metadata: {
name: "cluster-autoscaler",
namespace: "kube-system",
labels: {
app: "cluster-autoscaler",
},
annotations: {
"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
},
},
spec: {
replicas: 1,
selector: {
matchLabels: {
app: "cluster-autoscaler",
},
},
template: {
metadata: {
labels: {
app: "cluster-autoscaler",
},
annotations: {
"prometheus.io/scrape": "true",
"prometheus.io/port": "8085",
},
},
spec: {
serviceAccountName: "cluster-autoscaler",
containers: [
{
image: "k8s.gcr.io/autoscaling/cluster-autoscaler:" + version,
name: "cluster-autoscaler",
resources: {
limits: {
cpu: "100m",
memory: "300Mi",
},
requests: {
cpu: "100m",
memory: "300Mi",
},
},
command: [
"./cluster-autoscaler",
"--v=4",
"--stderrthreshold=info",
"--cloud-provider=aws",
"--skip-nodes-with-local-storage=false",
"--expander=least-waste",
"--node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/" +
cluster.clusterName,
"--balance-similar-node-groups",
"--skip-nodes-with-system-pods=false",
],
volumeMounts: [
{
name: "ssl-certs",
mountPath: "/etc/ssl/certs/ca-certificates.crt",
readOnly: true,
},
],
imagePullPolicy: "Always",
},
],
volumes: [
{
name: "ssl-certs",
hostPath: {
path: "/etc/ssl/certs/ca-bundle.crt",
},
},
],
},
},
},
},
],
});
}
}
@juanjoDiaz

This comment has been minimized.

Copy link

@juanjoDiaz juanjoDiaz commented Nov 19, 2020

Thanks for the useful script.

One note: the RoleBinding block seems duplicated.

  {
    "apiVersion": "rbac.authorization.k8s.io/v1",
    "kind": "RoleBinding",
    "metadata": {
      "name": "cluster-autoscaler",
      "namespace": "kube-system",
      "labels": {
        "k8s-addon": "cluster-autoscaler.addons.k8s.io",
        "k8s-app": "cluster-autoscaler"
      }
    },
    "roleRef": {
      "apiGroup": "rbac.authorization.k8s.io",
      "kind": "Role",
      "name": "cluster-autoscaler"
    },
    "subjects": [
      {
        "kind": "ServiceAccount",
        "name": "cluster-autoscaler",
        "namespace": "kube-system"
      }
    ]
  },
@esys

This comment has been minimized.

Copy link
Owner Author

@esys esys commented Nov 20, 2020

Thanks @juanjoDiaz ! Fixed. Too much json I guess ;)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment