Running Kubernetes Example on Fedora Atomic with flannel

README.md

Running Kubernetes Example on Fedora Atomic with flannel

It works same as Running Kubernetes Example on CoreOS, Part 2 with flannel (formerly Rudder).

Ref.) https://github.com/GoogleCloudPlatform/kubernetes/tree/master/docs/getting-started-guides/coreos
Ref.) https://github.com/GoogleCloudPlatform/kubernetes/tree/master/examples/guestbook

Step Zero: Build up a Kubernetes cluster

$ git clone https://github.com/YungSang/fedora-atomic-kubernetes-flannel.git
$ vagrant up

It will boot up one for etcd discovery, one master and two minion servers (minion-x).

Setup an SSH tunnel

Setup an SSH tunnel to the Kubernetes API Server in order to use kubecfg on your local machine.

$ curl -OL https://github.com/GoogleCloudPlatform/kubernetes/releases/download/v0.5.2/kubernetes.tar.gz
$ tar zxvf kubernetes.tar.gz kubernetes/platforms/darwin/amd64/kubecfg
x ./kubernetes/platforms/darwin/amd64/kubecfg
$ cp ./kubernetes/platforms/darwin/amd64/kubecfg /usr/local/bin
$ kubecfg -version
Kubernetes v0.5.2

$ vagrant ssh-config master > ssh.config
$ ssh -f -nNT -L 8080:127.0.0.1:8080 -F ssh.config master
$ kubecfg list pods
ID                  Image(s)            Host                Labels              Status
----------          ----------          ----------          ----------          ----------

Step One to Five: same as CoreOS

Running Kubernetes Example on CoreOS, Part 2 with flannel (formerly Rudder)

$ open http://192.168.12.11:8000
$ open http://192.168.12.12:8000

discovery.sh

#!/bin/sh

ADDR="%ADDR%"

cat <<EOF > /etc/systemd/system/etcd.service
[Unit]
Description=etcd

[Service]
Environment=ETCD_DATA_DIR=/var/lib/etcd
Environment=ETCD_NAME=%m
ExecStart=/usr/bin/etcd \
  -addr=${ADDR}:4001 \
  -peer-addr=${ADDR}:7001
Restart=always
RestartSec=10s
EOF
systemctl daemon-reload
systemctl start etcd.service

frontend-controller.json

{
  "id": "frontendController",
  "kind": "ReplicationController",
  "apiVersion": "v1beta1",
  "desiredState": {
    "replicas": 2,
    "replicaSelector": {"name": "frontend"},
    "podTemplate": {
      "desiredState": {
         "manifest": {
           "version": "v1beta1",
           "id": "frontendController",
           "containers": [{
             "name": "php-redis",
             "image": "brendanburns/php-redis",
             "cpu": 100,
             "memory": 50000000,
             "ports": [{"containerPort": 80, "hostPort": 8000}]
           }]
         }
       },
       "labels": {
         "name": "frontend",
         "uses": "redisslave,redis-master"
       }
      }},
  "labels": {"name": "frontend"}
}

master.sh

#!/bin/sh

ADDR="%ADDR%"
ETCD_DISCVERY="%ETCD_DISCVERY%"
MINION_IP_ADDRS="%MINION_IP_ADDRS%"

cd /tmp
curl -OLs https://github.com/coreos/etcd/releases/download/v0.4.6/etcd-v0.4.6-linux-amd64.tar.gz
tar zxvf etcd-v0.4.6-linux-amd64.tar.gz
cp etcd-v0.4.6-linux-amd64/etcdctl /opt/bin

cat <<EOF > /etc/systemd/system/etcd.service
[Unit]
Description=etcd

[Service]
Environment=ETCD_DATA_DIR=/var/lib/etcd
Environment=ETCD_NAME=%m
ExecStart=/usr/bin/etcd \
  -addr=${ADDR}:4001 \
  -peer-addr=${ADDR}:7001 \
  -discovery=http://${ETCD_DISCVERY}:4001/v2/keys/cluster
Restart=always
RestartSec=10
EOF
systemctl daemon-reload
systemctl start etcd.service

cat <<EOF > /etc/systemd/system/flannel.service
[Unit]
Requires=etcd.service
After=etcd.service

[Service]
ExecStartPre=/opt/bin/etcdctl set /coreos.com/network/config '{"Network":"10.100.0.0/16"}'
ExecStart=/opt/bin/flanneld -iface=${ADDR}
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable flannel.service
systemctl start flannel.service

systemctl disable docker.service
cat <<EOF > /etc/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.com
After=network.target docker.socket flannel.service
Requires=docker.socket flannel.service

[Service]
Type=notify
EnvironmentFile=-/etc/sysconfig/docker
EnvironmentFile=-/etc/sysconfig/docker-storage
EnvironmentFile=/run/flannel/subnet.env
ExecStartPre=-/usr/sbin/ip link set dev docker0 down
ExecStartPre=-/usr/sbin/ip link del dev docker0
ExecStart=/usr/bin/docker -d -H fd:// \$OPTIONS \$DOCKER_STORAGE_OPTIONS \
  --bip=\${FLANNEL_SUBNET} \
  --mtu=\${FLANNEL_MTU}
LimitNOFILE=1048576
LimitNPROC=1048576
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable docker.service
systemctl restart docker.service

iptables -I INPUT 1 -p tcp --dport 8080 -j ACCEPT -m comment --comment "kube-apiserver"

cat <<EOF > /etc/systemd/system/kube-apiserver.service
[Unit]
ConditionFileIsExecutable=/usr/bin/kube-apiserver
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
Wants=etcd.service
After=etcd.service

[Service]
ExecStart=/usr/bin/kube-apiserver \
  -address=127.0.0.1 \
  -port=8080 \
  -etcd_servers=http://127.0.0.1:4001 \
  -portal_net=10.100.0.0/16 \
  -logtostderr=true
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-apiserver.service
systemctl start kube-apiserver.service

cat <<EOF > /etc/systemd/system/kube-scheduler.service
[Unit]
ConditionFileIsExecutable=/usr/bin/kube-scheduler
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
Wants=kube-apiserver.service
After=kube-apiserver.service

[Service]
ExecStart=/usr/bin/kube-scheduler \
  -logtostderr=true \
  -master=127.0.0.1:8080
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-scheduler.service
systemctl start kube-scheduler.service

cat <<EOF > /etc/systemd/system/kube-controller-manager.service
[Unit]
ConditionFileIsExecutable=/usr/bin/kube-controller-manager
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
Wants=etcd.service
After=etcd.service

[Service]
ExecStart=/usr/bin/kube-controller-manager \
  -master=127.0.0.1:8080 \
  -machines=${MINION_IP_ADDRS} \
  -logtostderr=true
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-controller-manager.service
systemctl start kube-controller-manager.service

minion.sh

#!/bin/sh

ADDR="%ADDR%"
ETCD_DISCVERY="%ETCD_DISCVERY%"

cat <<EOF > /etc/systemd/system/etcd.service
[Unit]
Description=etcd

[Service]
Environment=ETCD_DATA_DIR=/var/lib/etcd
Environment=ETCD_NAME=%m
ExecStart=/usr/bin/etcd \
  -addr=${ADDR}:4001 \
  -peer-addr=${ADDR}:7001 \
  -discovery=http://${ETCD_DISCVERY}:4001/v2/keys/cluster
Restart=always
RestartSec=10s
EOF
systemctl daemon-reload
systemctl start etcd.service

cat <<EOF > /etc/systemd/system/flannel.service
[Unit]
Requires=etcd.service
After=etcd.service

[Service]
ExecStart=/opt/bin/flanneld -iface=${ADDR}

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable flannel.service
systemctl start flannel.service

systemctl disable docker.service
cat <<EOF > /etc/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.com
After=network.target docker.socket flannel.service
Requires=docker.socket flannel.service

[Service]
Type=notify
EnvironmentFile=-/etc/sysconfig/docker
EnvironmentFile=-/etc/sysconfig/docker-storage
EnvironmentFile=/run/flannel/subnet.env
ExecStartPre=-/usr/sbin/ip link set dev docker0 down
ExecStartPre=-/usr/sbin/ip link del dev docker0
ExecStart=/usr/bin/docker -d -H fd:// \$OPTIONS \$DOCKER_STORAGE_OPTIONS \
  --bip=\${FLANNEL_SUBNET} \
  --mtu=\${FLANNEL_MTU}
LimitNOFILE=1048576
LimitNPROC=1048576
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable docker.service
systemctl restart docker.service

iptables -I INPUT 1 -p tcp --dport 10250 -j ACCEPT -m comment --comment "kubelet"

cat <<EOF > /etc/systemd/system/kubelet.service
[Unit]
ConditionFileIsExecutable=/usr/bin/kubelet
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
Wants=etcd.service
After=etcd.service

[Service]
ExecStart=/usr/bin/kubelet \
  -address=0.0.0.0 \
  -port=10250 \
  -hostname_override=${ADDR} \
  -etcd_servers=http://127.0.0.1:4001 \
  -logtostderr=true
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kubelet.service
systemctl start kubelet.service

cat <<EOF > /etc/systemd/system/kube-proxy.service
[Unit]
ConditionFileIsExecutable=/usr/bin/kube-proxy
Description=Kubernetes Proxy
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
Wants=etcd.service
After=etcd.service

[Service]
ExecStart=/usr/bin/kube-proxy \
  -etcd_servers=http://127.0.0.1:4001 \
  -logtostderr=true
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-proxy.service
systemctl start kube-proxy.service

redis-master-service.json

{
  "id": "redis-master",
  "kind": "Service",
  "apiVersion": "v1beta1",
  "port": 6379,
  "containerPort": 6379,
  "selector": {
    "name": "redis-master"
  },
  "labels": {
    "name": "redis-master"
  }
}

redis-master.json

{
  "id": "redis-master",
  "kind": "Pod",
  "apiVersion": "v1beta1",
  "desiredState": {
    "manifest": {
      "version": "v1beta1",
      "id": "redis-master",
      "containers": [{
        "name": "master",
        "image": "dockerfile/redis",
        "cpu": 100,
        "ports": [{
          "containerPort": 6379,
          "hostPort": 6379
        }]
      }]
    }
  },
  "labels": {
    "name": "redis-master"
  }
}

redis-slave-controller.json

{
  "id": "redisSlaveController",
  "kind": "ReplicationController",
  "apiVersion": "v1beta1",
  "desiredState": {
    "replicas": 2,
    "replicaSelector": {"name": "redisslave"},
    "podTemplate": {
      "desiredState": {
         "manifest": {
           "version": "v1beta1",
           "id": "redisSlaveController",
           "containers": [{
             "name": "slave",
             "image": "brendanburns/redis-slave",
             "cpu": 200,
             "ports": [{"containerPort": 6379, "hostPort": 6380}]
           }]
         }
       },
       "labels": {
         "name": "redisslave",
         "uses": "redis-master",
       }
      }},
  "labels": {"name": "redisslave"}
}

redis-slave-service.json

{
  "id": "redisslave",
  "kind": "Service",
  "apiVersion": "v1beta1",
  "port": 6379,
  "containerPort": 6379,
  "labels": {
    "name": "redisslave"
  },
  "selector": {
    "name": "redisslave"
  }
}

Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2"

NUM_MINIONS = 2

BASE_IP_ADDR    = ENV['BASE_IP_ADDR'] || "192.168.12"
ETCD_DISCVERY   = "#{BASE_IP_ADDR}.101"
MASTER_IP_ADDR  = "#{BASE_IP_ADDR}.10"
MINION_IP_ADDRS = NUM_MINIONS.times.collect { |i| BASE_IP_ADDR + ".#{i+11}" }

Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  config.vm.box = "yungsang/fedora-atomic"

  config.vm.box_version = ">= 1.4.0"

  config.vm.define "discovery" do |discovery|
    discovery.vm.hostname = "discovery"

    discovery.vm.network :private_network, ip: ETCD_DISCVERY

    discovery.vm.provision :file, source: "./discovery.sh", destination: "/tmp/user-data.sh"

    discovery.vm.provision :shell do |sh|
      sh.privileged = true
      sh.inline = <<-EOT
        sed -e "s/%ADDR%/#{ETCD_DISCVERY}/g" -i /tmp/user-data.sh

        chmod +x /tmp/user-data.sh
        /tmp/user-data.sh
      EOT
    end
  end

  config.vm.define "master" do |master|
    master.vm.hostname = "master"

    master.vm.network :forwarded_port, guest: 4001, host: 4001

    master.vm.network :private_network, ip: MASTER_IP_ADDR

    # Install flannel
    master.vm.provision :docker do |d|
      d.run "yungsang/flannel",
        args: "--rm -v /opt/bin:/target --privileged",
        auto_assign_name: false, daemonize: false
    end

    master.vm.provision :file, source: "./master.sh", destination: "/tmp/user-data.sh"

    master.vm.provision :shell do |sh|
      sh.privileged = true
      sh.inline = <<-EOT
        sed -e "s/%ADDR%/#{MASTER_IP_ADDR}/g" -i /tmp/user-data.sh
        sed -e "s/%ETCD_DISCVERY%/#{ETCD_DISCVERY}/g" -i /tmp/user-data.sh
        sed -e "s/%MINION_IP_ADDRS%/#{MINION_IP_ADDRS.join(',')}/g" -i /tmp/user-data.sh

        chmod +x /tmp/user-data.sh
        /tmp/user-data.sh
      EOT
    end
  end

  NUM_MINIONS.times do |i|
    config.vm.define "minion-#{i+1}" do |minion|
      minion.vm.hostname = "minion-#{i+1}"

      minion.vm.network :private_network, ip: MINION_IP_ADDRS[i]

      # Install flannel
      minion.vm.provision :docker do |d|
        d.run "yungsang/flannel",
          args: "--rm -v /opt/bin:/target --privileged",
          auto_assign_name: false, daemonize: false
      end

      minion.vm.provision :file, source: "./minion.sh", destination: "/tmp/user-data.sh"

      minion.vm.provision :shell do |sh|
        sh.privileged = true
        sh.inline = <<-EOT
          sed -e "s/%ADDR%/#{MINION_IP_ADDRS[i]}/g" -i /tmp/user-data.sh
          sed -e "s/%ETCD_DISCVERY%/#{ETCD_DISCVERY}/g" -i /tmp/user-data.sh

          chmod +x /tmp/user-data.sh
          /tmp/user-data.sh
        EOT
      end
    end
  end
end

It has been transferred to https://github.com/YungSang/fedora-atomic-kubernetes-flannel .