Skip to content

Instantly share code, notes, and snippets.

@zedalaye
Created July 17, 2015 13:56
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save zedalaye/3f235429b0a7b86d9584 to your computer and use it in GitHub Desktop.
Save zedalaye/3f235429b0a7b86d9584 to your computer and use it in GitHub Desktop.
My deix cloudconfig for coreos running on xen
#cloud-config
---
hostname: %XSVMNAMETOHOSTNAME%
ssh_authorized_keys:
# - ssh-rsa <my key !>
# The following entry will automatically be replaced with a public key
# generated by XenServer's container management. The key-entry must exist,
# in order to enable container management for this VM.
- ssh-rsa %XSCONTAINERRSAPUB%
coreos:
etcd:
# generate a new token for each unique cluster from https://discovery.etcd.io/new
# uncomment the following line and replace it with your discovery URL
# discovery: https://discovery.etcd.io/<my cluster token>
addr: 192.168.57.31:4001
peer-addr: 192.168.57.31:7001
# give etcd more time if it's under heavy load - prevent leader election thrashing
peer-election-timeout: 2000
# heartbeat interval should ideally be 1/4 or 1/5 of peer election timeout
peer-heartbeat-interval: 500
fleet:
# We have to set the public_ip here so this works on Vagrant -- otherwise, Vagrant VMs
# will all publish the same private IP. This is harmless for cloud providers.
public-ip: 192.168.57.31
# allow etcd to slow down at times
etcd_request_timeout: 3.0
units:
- name: etcd.service
command: start
- name: 00-eth0.network
runtime: true
content: |
[Match]
Name=eth0
[Network]
DNS=192.168.57.238
Address=192.168.57.31/24
Gateway=192.168.57.1
# XenServer Linux Guest Agent
- name: xe-linux-distribution.service
command: start
content: |
[Unit]
Description=XenServer Linux Guest Agent
After=docker.service
[Service]
ExecStartPre=/media/configdrive/agent/xe-linux-distribution /var/cache/xe-linux-distribution
Environment="XE_UPDATE_GUEST_ATTRS=/media/configdrive/agent/xe-update-guest-attrs"
ExecStart=/media/configdrive/agent/xe-daemon
- name: docker-tcp.socket
command: start
enable: true
content: |
[Unit]
Description=Docker Socket for the API
[Socket]
ListenStream=2375
BindIPv6Only=both
Service=docker.service
[Install]
WantedBy=sockets.target
- name: stop-update-engine.service
command: start
content: |
[Unit]
Description=stop update-engine
[Service]
Type=oneshot
ExecStart=/usr/bin/systemctl stop update-engine.service
ExecStartPost=/usr/bin/systemctl mask update-engine.service
- name: graceful-deis-shutdown.service
content: |
[Unit]
Description=Clean up
DefaultDependencies=no
After=fleet.service etcd.service docker.service docker.socket deis-store-admin.service deis-store-daemon.service deis-store-volume.service deis-store-monitor.service
Requires=fleet.service etcd.service deis-store-admin.service deis-store-daemon.service deis-store-volume.service docker.service docker.socket deis-store-monitor.service
[Install]
WantedBy=shutdown.target halt.target reboot.target
[Service]
ExecStop=/opt/bin/graceful-shutdown.sh --really
Type=oneshot
TimeoutSec=1200
RemainAfterExit=yes
- name: install-deisctl.service
command: start
content: |
[Unit]
Description=Install deisctl utility
ConditionPathExists=!/opt/bin/deisctl
[Service]
Type=oneshot
ExecStart=/usr/bin/sh -c 'curl -sSL --retry 5 --retry-delay 2 http://deis.io/deisctl/install.sh | sh -s 1.8.0'
- name: debug-etcd.service
content: |
[Unit]
Description=etcd debugging service
[Service]
ExecStartPre=/usr/bin/curl -sSL -o /opt/bin/jq http://stedolan.github.io/jq/download/linux64/jq
ExecStartPre=/usr/bin/chmod +x /opt/bin/jq
ExecStart=/usr/bin/bash -c "while true; do curl -sL http://127.0.0.1:4001/v2/stats/leader | /opt/bin/jq . ; sleep 1 ; done"
- name: increase-nf_conntrack-connections.service
command: start
content: |
[Unit]
Description=Increase the number of connections in nf_conntrack. default is 65536
[Service]
Type=oneshot
ExecStartPre=/usr/sbin/modprobe nf_conntrack
ExecStart=/bin/sh -c "sysctl -w net.netfilter.nf_conntrack_max=262144"
- name: load-overlay-module.service
command: start
content: |
[Unit]
Description=Load overlay module before docker start
Before=docker.service
[Service]
ExecStart=/bin/bash -c "lsmod | grep overlay || modprobe overlay"
- name: fleet.service
command: start
write_files:
# Enable ARP notifications for smooth network recovery after migrations
- path: /etc/sysctl.d/10-enable-arp-notify.conf
permissions: 0644
owner: root
content: |
net.ipv4.conf.all.arp_notify = 1
- path: /etc/environment
permissions: 0644
content: |
COREOS_PUBLIC_IPV4=192.168.57.31
COREOS_PRIVATE_IPV4=192.168.57.31
- path: /etc/deis-release
content: |
DEIS_RELEASE=v1.8.0
- path: /etc/motd
content: " \e[31m* * \e[34m* \e[32m***** \e[39mddddd eeeeeee iiiiiii ssss\n\e[31m* * \e[34m* * \e[32m* * \e[39md d e e i s s\n \e[31m* * \e[34m***** \e[32m***** \e[39md d e i s\n\e[32m***** \e[31m* * \e[34m* \e[39md d e i s\n\e[32m* * \e[31m* * \e[34m* * \e[39md d eee i sss\n\e[32m***** \e[31m* * \e[34m***** \e[39md d e i s\n \e[34m* \e[32m***** \e[31m* * \e[39md d e i s\n \e[34m* * \e[32m* * \e[31m* * \e[39md d e e i s s\n\e[34m***** \e[32m***** \e[31m* * \e[39mddddd eeeeeee iiiiiii ssss\n\n\e[39mWelcome to Deis\t\t\tPowered by Core\e[38;5;45mO\e[38;5;206mS\e[39m\n"
- path: /etc/profile.d/nse-function.sh
permissions: '0755'
content: |
function nse() {
docker exec -it $1 bash
}
- path: /etc/systemd/system/docker.service.d/50-insecure-registry.conf
content: |
[Service]
EnvironmentFile=/etc/environment_proxy
Environment="DOCKER_OPTS=--insecure-registry 10.0.0.0/8 --insecure-registry 172.16.0.0/12 --insecure-registry 192.168.0.0/16 --insecure-registry 100.64.0.0/10"
- path: /run/deis/bin/get_image
permissions: '0755'
content: |
#!/bin/bash
# usage: get_image <component_path>
IMAGE=`etcdctl get $1/image 2>/dev/null`
# if no image was set in etcd, we use the default plus the release string
if [ $? -ne 0 ]; then
RELEASE=`etcdctl get /deis/platform/version 2>/dev/null`
# if no release was set in etcd, use the default provisioned with the server
if [ $? -ne 0 ]; then
source /etc/deis-release
RELEASE=$DEIS_RELEASE
fi
IMAGE=$1:$RELEASE
fi
# remove leading slash
echo ${IMAGE#/}
- path: /run/deis/bin/preseed
permissions: '0755'
content: |
#!/bin/bash
COMPONENTS=(builder cache controller database logger logspout publisher registry router store-daemon store-gateway store-metadata store-monitor)
for c in "${COMPONENTS[@]}"; do
image=`/run/deis/bin/get_image /deis/$c`
docker history $image >/dev/null 2>&1 || docker pull $image
done
- path: /opt/bin/deis-debug-logs
permissions: '0755'
content: |
#!/bin/bash
echo '--- VERSIONS ---'
source /etc/os-release
echo $PRETTY_NAME
source /etc/deis-release
echo "Deis $DEIS_RELEASE"
etcd -version
fleet -version
printf "\n"
echo '--- SYSTEM STATUS ---'
journalctl -n 50 -u etcd --no-pager
journalctl -n 50 -u fleet --no-pager
printf "\n"
echo '--- DEIS STATUS ---'
deisctl list
etcdctl ls --recursive /deis
printf "\n"
- path: /home/core/.toolboxrc
owner: core
content: |
TOOLBOX_DOCKER_IMAGE=alpine
TOOLBOX_DOCKER_TAG=3.1
TOOLBOX_USER=root
- path: /etc/environment_proxy
owner: core
content: |
HTTP_PROXY=
HTTPS_PROXY=
ALL_PROXY=
NO_PROXY=
http_proxy=
https_proxy=
all_proxy=
no_proxy=
- path: /etc/systemd/coredump.conf
content: |
[Coredump]
Storage=none
- path: /opt/bin/graceful-shutdown.sh
permissions: '0755'
content: |
#!/usr/bin/bash
if [ "$1" != '--really' ]; then
echo "command must be run as: $0 --really"
exit 1
fi
# procedure requires the store-admin
ADMIN_RUNNING=$(docker inspect --format="{{ .State.Running }}" deis-store-admin)
if [ $? -eq 1 ] || [ "$ADMIN_RUNNING" == "false" ]; then
echo "deis-store-admin container is required for graceful shutdown"
exit 2
fi
set -e -x -o pipefail
# determine osd id
CURRENT_STATUS=$(/usr/bin/docker exec deis-store-admin ceph health | awk '{print $1}')
OSD_HOSTS=($(/usr/bin/etcdctl ls /deis/store/hosts/| awk -F'/' '{print $5}'))
for HOST in "${OSD_HOSTS[@]}"
do
PUBLIC_IP=$(fleetctl list-machines -fields="machine,ip" -full -no-legend| grep `cat /etc/machine-id` | awk '{print $2}')
if [ "$HOST" = "$PUBLIC_IP" ] ; then
OSD_ID=$(/usr/bin/etcdctl get /deis/store/osds/$PUBLIC_IP)
break
fi
done
# if we own an osd and its healthy, try to gracefully remove it
if [ ! -z "$OSD_ID" ] && [[ "$CURRENT_STATUS" == *"HEALTH_OK"* ]] && [ ${#OSD_HOSTS[@]} -gt "3" ]; then
/usr/bin/docker exec deis-store-admin ceph osd out $OSD_ID
sleep 30
TIMEWAITED=0
until [[ $(/usr/bin/docker exec deis-store-admin ceph health) == *"HEALTH_OK"* ]]
do
if [ $TIMEWAITED -gt "1200" ]
then
echo "ceph graceful removal timeout exceeded"
break
fi
echo "waiting" && sleep 5
TIMEWAITED=$((TIMEWAITED+5))
done
/usr/bin/docker stop deis-store-daemon
/usr/bin/docker exec deis-store-admin ceph osd crush remove osd.$OSD_ID
/usr/bin/docker exec deis-store-admin ceph auth del osd.$OSD_ID
/usr/bin/docker exec deis-store-admin ceph osd rm $OSD_ID
/usr/bin/etcdctl rm /deis/store/osds/$PUBLIC_IP
etcdctl rm /deis/store/hosts/$PUBLIC_IP && sleep 10
# remove ceph mon
/usr/bin/docker stop deis-store-monitor || true
/usr/bin/docker exec deis-store-admin ceph mon remove `hostname -f` # fixme
/usr/bin/docker stop deis-store-metadata || true
fi
NODE=$(curl -L http://127.0.0.1:7001/v2/admin/machines/`cat /etc/machine-id`)
# remove from etcd cluster
if [ $NODE != 'null' ]; then
/usr/bin/curl -L -XDELETE http://127.0.0.1:7001/v2/admin/machines/`cat /etc/machine-id`
fi
manage_etc_hosts: localhost
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment