Utilize Jenkins in an auto-scaling Kubernetes deployment on Amazon EKS
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
labels: | |
k8s-addon: cluster-autoscaler.addons.k8s.io | |
k8s-app: cluster-autoscaler | |
name: cluster-autoscaler | |
namespace: kube-system | |
--- | |
apiVersion: rbac.authorization.k8s.io/v1beta1 | |
kind: ClusterRole | |
metadata: | |
name: cluster-autoscaler | |
labels: | |
k8s-addon: cluster-autoscaler.addons.k8s.io | |
k8s-app: cluster-autoscaler | |
rules: | |
- apiGroups: [""] | |
resources: ["events","endpoints"] | |
verbs: ["create", "patch"] | |
- apiGroups: [""] | |
resources: ["pods/eviction"] | |
verbs: ["create"] | |
- apiGroups: [""] | |
resources: ["pods/status"] | |
verbs: ["update"] | |
- apiGroups: [""] | |
resources: ["endpoints"] | |
resourceNames: ["cluster-autoscaler"] | |
verbs: ["get","update"] | |
- apiGroups: [""] | |
resources: ["nodes"] | |
verbs: ["watch","list","get","update"] | |
- apiGroups: [""] | |
resources: ["pods","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"] | |
verbs: ["watch","list","get"] | |
- apiGroups: ["extensions"] | |
resources: ["replicasets","daemonsets"] | |
verbs: ["watch","list","get"] | |
- apiGroups: ["policy"] | |
resources: ["poddisruptionbudgets"] | |
verbs: ["watch","list"] | |
- apiGroups: ["batch"] | |
resources: ["jobs"] | |
verbs: ["watch", "list", "get"] | |
- apiGroups: ["apps"] | |
resources: ["statefulsets", "replicasets", "daemonsets"] | |
verbs: ["watch","list","get"] | |
- apiGroups: ["storage.k8s.io"] | |
resources: ["storageclasses"] | |
verbs: ["watch","list","get"] | |
--- | |
apiVersion: rbac.authorization.k8s.io/v1beta1 | |
kind: Role | |
metadata: | |
name: cluster-autoscaler | |
namespace: kube-system | |
labels: | |
k8s-addon: cluster-autoscaler.addons.k8s.io | |
k8s-app: cluster-autoscaler | |
rules: | |
- apiGroups: [""] | |
resources: ["configmaps"] | |
verbs: ["create"] | |
- apiGroups: [""] | |
resources: ["configmaps"] | |
resourceNames: ["cluster-autoscaler-status"] | |
verbs: ["delete","get","update"] | |
--- | |
apiVersion: rbac.authorization.k8s.io/v1beta1 | |
kind: ClusterRoleBinding | |
metadata: | |
name: cluster-autoscaler | |
labels: | |
k8s-addon: cluster-autoscaler.addons.k8s.io | |
k8s-app: cluster-autoscaler | |
roleRef: | |
apiGroup: rbac.authorization.k8s.io | |
kind: ClusterRole | |
name: cluster-autoscaler | |
subjects: | |
- kind: ServiceAccount | |
name: cluster-autoscaler | |
namespace: kube-system | |
--- | |
apiVersion: rbac.authorization.k8s.io/v1beta1 | |
kind: RoleBinding | |
metadata: | |
name: cluster-autoscaler | |
namespace: kube-system | |
labels: | |
k8s-addon: cluster-autoscaler.addons.k8s.io | |
k8s-app: cluster-autoscaler | |
roleRef: | |
apiGroup: rbac.authorization.k8s.io | |
kind: Role | |
name: cluster-autoscaler | |
subjects: | |
- kind: ServiceAccount | |
name: cluster-autoscaler | |
namespace: kube-system | |
--- | |
apiVersion: apps/v1 | |
kind: Deployment | |
metadata: | |
name: cluster-autoscaler | |
namespace: kube-system | |
labels: | |
app: cluster-autoscaler | |
spec: | |
replicas: 1 | |
selector: | |
matchLabels: | |
app: cluster-autoscaler | |
template: | |
metadata: | |
labels: | |
app: cluster-autoscaler | |
spec: | |
serviceAccountName: cluster-autoscaler | |
containers: | |
- image: k8s.gcr.io/cluster-autoscaler:v1.14.6 | |
name: cluster-autoscaler | |
resources: | |
limits: | |
cpu: 100m | |
memory: 300Mi | |
requests: | |
cpu: 100m | |
memory: 300Mi | |
command: | |
- ./cluster-autoscaler | |
- --v=4 | |
- --stderrthreshold=info | |
- --cloud-provider=aws | |
- --skip-nodes-with-local-storage=false | |
- --nodes=0:5:YOUR_SPOT_ASG_NODEGROUP | |
env: | |
- name: AWS_REGION | |
value: YOUR_SPOT_ASG_AWS_REGION | |
volumeMounts: | |
- name: ssl-certs | |
mountPath: /etc/ssl/certs/ca-certificates.crt | |
readOnly: true | |
imagePullPolicy: "Always" | |
volumes: | |
- name: ssl-certs | |
hostPath: | |
path: "/etc/ssl/certs/ca-bundle.crt" | |
FROM jenkins/jenkins:2.204.5 | |
RUN /usr/local/bin/install-plugins.sh ssh-slaves | |
RUN /usr/local/bin/install-plugins.sh kubernetes | |
USER root | |
RUN apt-get update && apt-get install -y maven | |
USER jenkins |
FROM ubuntu:18.04 | |
# Setup JNLP for jenkins slave | |
RUN apt-get update && \ | |
apt-get install -y apt-transport-https curl init openssh-server openssh-client openjdk-8-jdk sudo | |
RUN groupadd -g 10000 jenkins && \ | |
useradd -d $HOME -u 10000 -g jenkins jenkins && \ | |
adduser jenkins sudo | |
RUN curl --create-dirs -sSLo /usr/share/jenkins/slave.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/3.40/remoting-3.40.jar && \ | |
chmod 755 /usr/share/jenkins && \ | |
chmod 644 /usr/share/jenkins/slave.jar | |
COPY jenkins-slave /usr/local/bin/jenkins-slave | |
RUN chmod 755 /usr/local/bin/jenkins-slave && \ | |
chown jenkins:jenkins /usr/local/bin/jenkins-slave | |
RUN mkdir -p /home/jenkins/.jenkins && \ | |
mkdir -p /home/jenkins/agent && \ | |
chown -R jenkins:jenkins /home/jenkins | |
VOLUME /home/jenkins/.jenkins | |
VOLUME /home/jenkins/agent | |
WORKDIR /home/jenkins | |
ENTRYPOINT ["/usr/local/bin/jenkins-slave"] |
apiVersion: eksctl.io/v1alpha5 | |
kind: ClusterConfig | |
metadata: | |
name: eks-test-jenkins | |
region: YOUR_CLUSTER_AWS_REGION | |
vpc: | |
subnets: | |
private: | |
us-east-1a: | |
id: "YOUR_SUBNET_1_ID" | |
us-east-1b: | |
id: "YOUR_SUBNET_2_ID" | |
nodeGroups: | |
- name: ng-test-jenkins | |
instanceType: m5.xlarge | |
desiredCapacity: 1 | |
privateNetworking: true | |
- name: ng-test-jenkins-spot | |
minSize: 0 | |
maxSize: 5 | |
desiredCapacity: 0 | |
privateNetworking: true | |
instancesDistribution: | |
instanceTypes: ["m5.4xlarge", "m4.4xlarge", "r5.4xlarge", "r5a.4xlarge", "m5a.4xlarge", "r4.4xlarge"] | |
onDemandBaseCapacity: 0 | |
onDemandPercentageAboveBaseCapacity: 0 | |
spotInstancePools: 4 | |
labels: | |
lifecycle: Ec2Spot | |
intent: apps | |
taints: | |
spotInstance: "true:PreferNoSchedule" | |
tags: | |
k8s.io/cluster-autoscaler/node-template/label/lifecycle: Ec2Spot | |
k8s.io/cluster-autoscaler/node-template/label/intent: apps | |
k8s.io/cluster-autoscaler/node-template/taint/spotInstance: "true:PreferNoSchedule" | |
iam: | |
withAddonPolicies: | |
autoScaler: true | |
cloudWatch: true | |
albIngress: true |
apiVersion: extensions/v1beta1 | |
kind: Deployment | |
metadata: | |
name: jenkins | |
spec: | |
replicas: 1 | |
template: | |
metadata: | |
labels: | |
app: jenkins | |
spec: | |
containers: | |
- name: jenkins | |
image: YOUR_JENKINS_DOCKER_IMAGE | |
env: | |
- name: JAVA_OPTS | |
value: -Djenkins.install.runSetupWizard=false | |
ports: | |
- name: http-port | |
containerPort: 8080 | |
- name: jnlp-port | |
containerPort: 50000 | |
volumeMounts: | |
- name: jenkins-home | |
mountPath: /var/jenkins_home | |
initContainers: | |
- name: set-jenkins-home-permissions | |
image: alpine:3.11 | |
command: | |
- chown | |
- -R | |
- 1000:1000 | |
- /var/jenkins_home | |
volumeMounts: | |
- name: jenkins-home | |
mountPath: /var/jenkins_home | |
volumes: | |
- name: jenkins-home | |
persistentVolumeClaim: | |
claimName: eks-test-jenkins-pv-claim |
#!/usr/bin/env sh | |
# The MIT License | |
# | |
# Copyright (c) 2015, CloudBees, Inc. | |
# | |
# Permission is hereby granted, free of charge, to any person obtaining a copy | |
# of this software and associated documentation files (the "Software"), to deal | |
# in the Software without restriction, including without limitation the rights | |
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
# copies of the Software, and to permit persons to whom the Software is | |
# furnished to do so, subject to the following conditions: | |
# | |
# The above copyright notice and this permission notice shall be included in | |
# all copies or substantial portions of the Software. | |
# | |
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | |
# THE SOFTWARE. | |
# Usage jenkins-slave.sh [options] -url http://jenkins [SECRET] [AGENT_NAME] | |
# Optional environment variables : | |
# * JENKINS_TUNNEL : HOST:PORT for a tunnel to route TCP traffic to jenkins host, when jenkins can't be directly accessed over network | |
# * JENKINS_URL : alternate jenkins URL | |
# * JENKINS_SECRET : agent secret, if not set as an argument | |
# * JENKINS_AGENT_NAME : agent name, if not set as an argument | |
# * JENKINS_AGENT_WORKDIR : agent work directory, if not set by optional parameter -workDir | |
if [ $# -eq 1 ]; then | |
# if `docker run` only has one arguments, we assume user is running alternate command like `bash` to inspect the image | |
exec "$@" | |
else | |
# if -tunnel is not provided, try env vars | |
case "$@" in | |
*"-tunnel "*) ;; | |
*) | |
if [ ! -z "$JENKINS_TUNNEL" ]; then | |
TUNNEL="-tunnel $JENKINS_TUNNEL" | |
fi ;; | |
esac | |
# if -workDir is not provided, try env vars | |
if [ ! -z "$JENKINS_AGENT_WORKDIR" ]; then | |
case "$@" in | |
*"-workDir"*) echo "Warning: Work directory is defined twice in command-line arguments and the environment variable" ;; | |
*) | |
WORKDIR="-workDir $JENKINS_AGENT_WORKDIR" ;; | |
esac | |
fi | |
if [ -n "$JENKINS_URL" ]; then | |
URL="-url $JENKINS_URL" | |
fi | |
if [ -n "$JENKINS_NAME" ]; then | |
JENKINS_AGENT_NAME="$JENKINS_NAME" | |
fi | |
if [ -z "$JNLP_PROTOCOL_OPTS" ]; then | |
echo "Warning: JnlpProtocol3 is disabled by default, use JNLP_PROTOCOL_OPTS to alter the behavior" | |
JNLP_PROTOCOL_OPTS="-Dorg.jenkinsci.remoting.engine.JnlpProtocol3.disabled=true" | |
fi | |
# if java home is defined, use it | |
JAVA_BIN="java" | |
if [ "$JAVA_HOME" ]; then | |
JAVA_BIN="$JAVA_HOME/bin/java" | |
fi | |
# if both required options are defined, do not pass the parameters | |
OPT_JENKINS_SECRET="" | |
if [ -n "$JENKINS_SECRET" ]; then | |
case "$@" in | |
*"${JENKINS_SECRET}"*) echo "Warning: SECRET is defined twice in command-line arguments and the environment variable" ;; | |
*) | |
OPT_JENKINS_SECRET="${JENKINS_SECRET}" ;; | |
esac | |
fi | |
OPT_JENKINS_AGENT_NAME="" | |
if [ -n "$JENKINS_AGENT_NAME" ]; then | |
case "$@" in | |
*"${JENKINS_AGENT_NAME}"*) echo "Warning: AGENT_NAME is defined twice in command-line arguments and the environment variable" ;; | |
*) | |
OPT_JENKINS_AGENT_NAME="${JENKINS_AGENT_NAME}" ;; | |
esac | |
fi | |
#TODO: Handle the case when the command-line and Environment variable contain different values. | |
#It is fine it blows up for now since it should lead to an error anyway. | |
exec $JAVA_BIN $JAVA_OPTS $JNLP_PROTOCOL_OPTS -cp /usr/share/jenkins/slave.jar hudson.remoting.jnlp.Main -headless $TUNNEL $URL $WORKDIR $OPT_JENKINS_SECRET $OPT_JENKINS_AGENT_NAME "$@" | |
fi |
{ | |
"Version": "2012-10-17", | |
"Statement": [ | |
{ | |
"Effect": "Allow", | |
"Action": [ | |
"autoscaling:DescribeAutoScalingGroups", | |
"autoscaling:DescribeAutoScalingInstances", | |
"autoscaling:SetDesiredCapacity", | |
"autoscaling:TerminateInstanceInAutoScalingGroup", | |
"autoscaling:DescribeTags" | |
], | |
"Resource": "*" | |
} | |
] | |
} |
apiVersion: v1 | |
kind: PersistentVolumeClaim | |
metadata: | |
name: eks-test-jenkins-pv-claim | |
spec: | |
storageClassName: manual | |
accessModes: | |
- ReadWriteOnce | |
resources: | |
requests: | |
storage: 5Gi |
apiVersion: v1 | |
kind: PersistentVolume | |
metadata: | |
name: eks-test-jenkins-pv-volume | |
labels: | |
type: local | |
spec: | |
storageClassName: manual | |
capacity: | |
storage: 10Gi | |
accessModes: | |
- ReadWriteOnce | |
hostPath: | |
path: "/mnt/data" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment