Last active
November 17, 2023 08:04
-
-
Save camposdelima/c77a4f23a9a831188b88ca67650cf011 to your computer and use it in GitHub Desktop.
How remove spot taints from AKS nodepools and maintain the spot nodes without taints
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: apps/v1 | |
kind: DaemonSet | |
metadata: | |
name: spot-supervisor | |
labels: | |
app.kubernetes.io/instance: spot-supervisor | |
spec: | |
selector: | |
matchLabels: | |
name: spot-supervisor | |
template: | |
metadata: | |
labels: | |
name: spot-supervisor | |
spec: | |
restartPolicy: Always | |
priorityClassName: system-node-critical | |
containers: | |
- name: spot-supervisor | |
image: kubesphere/kubectl:v1.0.0 | |
command: ["/bin/sh","-c"] | |
env: | |
- name: NODE_NAME | |
valueFrom: | |
fieldRef: | |
apiVersion: v1 | |
fieldPath: spec.nodeName | |
args: | |
- >- | |
while sleep 10; | |
do | |
kubectl get ValidatingWebhookConfiguration aks-node-validating-webhook -o yaml | sed -e 's/\(objectSelector: \){}/\1{"matchLabels": {"disable":"true"}}/g' | kubectl apply -f -; | |
kubectl taint node -l kubernetes.azure.com/scalesetpriority=spot kubernetes.azure.com/scalesetpriority=spot:NoSchedule- || echo "spot done"; | |
echo checking events to $NODE_NAME... | |
( | |
( curl -H Metadata:true http://169.254.169.254/metadata/scheduledevents?api-version=2020-07-01 2>&1 | grep Preempt ) && | |
( echo "set taint to $NODE_NAME" && kubectl taint node $NODE_NAME kubernetes.azure.com/event=spot:NoExecute & | |
kubectl drain $NODE_NAME ) | |
) || echo "No event" | |
echo -e | |
done; | |
echo end; | |
tolerations: | |
- effect: NoSchedule | |
key: node-role.kubernetes.io/master | |
- key: "kubernetes.azure.com/scalesetpriority" | |
operator: "Equal" | |
value: "spot" | |
effect: "NoSchedule" | |
--- | |
apiVersion: rbac.authorization.k8s.io/v1 | |
kind: ClusterRole | |
metadata: | |
name: spot-supervisor | |
rules: | |
- apiGroups: | |
- "" | |
- admissionregistration.k8s.io | |
resources: | |
- nodes | |
- validatingwebhookconfigurations | |
verbs: ["get", "list","patch", "delete"] | |
--- | |
apiVersion: rbac.authorization.k8s.io/v1 | |
kind: ClusterRoleBinding | |
metadata: | |
name: spot-supervisor | |
subjects: | |
- kind: ServiceAccount | |
name: default | |
roleRef: | |
apiGroup: rbac.authorization.k8s.io | |
kind: ClusterRole | |
name: spot-supervisor |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment