Install, configure, start minikube

kube-dns.yaml

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Source: https://github.com/kubernetes/minikube/blob/master/deploy/addons/kube-dns/kube-dns-rc.yaml

apiVersion: v1
kind: ReplicationController
metadata:
  name: kube-dns-v20
  namespace: kube-system
  labels:
    k8s-app: kube-dns
    version: v20
    #kubernetes.io/cluster-service: "true"
spec:
  replicas: 1
  selector:
    k8s-app: kube-dns
    version: v20
  template:
    metadata:
      labels:
        k8s-app: kube-dns
        version: v20
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
    spec:
      containers:
      - name: kubedns
        image: gcr.io/google_containers/kubedns-amd64:1.9
        imagePullPolicy: IfNotPresent
        resources:
          # TODO: Set memory limits when we've profiled the container for large
          # clusters, then set request = limit to keep this container in
          # guaranteed class. Currently, this container falls into the
          # "burstable" category so the kubelet doesn't backoff from restarting it.
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        livenessProbe:
          httpGet:
            path: /healthz-kubedns
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /readiness
            port: 8081
            scheme: HTTP
          # we poll on pod startup for the Kubernetes master service and
          # only setup the /readiness HTTP server once that's available.
          initialDelaySeconds: 3
          timeoutSeconds: 5
        args:
        # command = "/kube-dns"
        - --domain=minikube.example.com.
        - --dns-port=10053
        ports:
        - containerPort: 10053
          name: dns-local
          protocol: UDP
        - containerPort: 10053
          name: dns-tcp-local
          protocol: TCP
      - name: dnsmasq
        image: gcr.io/google_containers/kube-dnsmasq-amd64:1.4
        livenessProbe:
          httpGet:
            path: /healthz-dnsmasq
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        args:
        - --cache-size=1000
        - --no-resolv
        - --server=127.0.0.1#10053
        - --log-facility=-
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
      - name: healthz
        image: gcr.io/google_containers/exechealthz-amd64:1.2
        resources:
          limits:
            memory: 50Mi
          requests:
            cpu: 10m
            # Note that this container shouldn't really need 50Mi of memory. The
            # limits are set higher than expected pending investigation on #29688.
            # The extra memory was stolen from the kubedns container to keep the
            # net memory requested by the pod constant.
            memory: 50Mi
        args:
        - --cmd=nslookup kubernetes.default.svc.minikube.example.com 127.0.0.1 >/dev/null
        - --url=/healthz-dnsmasq
        - --cmd=nslookup kubernetes.default.svc.minikube.example.com 127.0.0.1:10053 >/dev/null
        - --url=/healthz-kubedns
        - --port=8080
        - --quiet
        ports:
        - containerPort: 8080
          protocol: TCP
      dnsPolicy: Default  # Don't use cluster DNS.

---
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Source: https://github.com/kubernetes/minikube/blob/master/deploy/addons/kube-dns/kube-dns-svc.yaml

apiVersion: v1
kind: Service
metadata:
  name: kube-dns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
    #kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "KubeDNS"
spec:
  selector:
    k8s-app: kube-dns
  clusterIP: 10.0.0.10
  ports:
  - name: dns
    port: 53
    protocol: UDP
  - name: dns-tcp
    port: 53
    protocol: TCP

minikube-curlbash-to-the-extreme.sh

#!/bin/bash
set -eo pipefail

MINIKUBE_VERSION=v0.20.0

#MINIKUBE_PR_NUMBER=1388 # Fixed mount command to bind on specific VM ip address https://github.com/kubernetes/minikube/pull/1388

VM_DRIVER=${VM_DRIVER:-virtualbox}
KUBECTL_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)
DOCKER_SERVER="docker-registry.example.com"
DNS_DOMAIN="minikube.example.com"

function main () {
  log "installing/updating minikube"
  install_minikube_bin

  log "installing/updating kubectl"
  install_kubectl_bin

  [[ $VM_DRIVER == "xhyve" ]] && install_xhyve_machine_driver
  [[ $VM_DRIVER == "kvm" ]] && install_kvm_machine_driver

  if minikube_is_running; then
    log "minikube is already up; refreshing configuration"
    configure_minikube
  else
    log "(re)initializing local minikube environment"
    prompt_for_docker_credentials
    ensure_minikube_running
    configure_minikube
  fi
}

function log() {
  echo "$( date +%Y-%m-%d\ %H:%M:%S.%N\ %Z )  LOG  $*" 1>&2
}

function die() {
  echo "$( date +%Y-%m-%d\ %H:%M:%S.%N\ %Z )  ERROR  $*" 1>&2
  exit 1
}

function minikube_is_running () {
  local minikube_status=$(minikube status --format '{{.MinikubeStatus}}')
  local localkube_status=$(minikube status --format '{{.LocalkubeStatus}}')
  if [[ "${minikube_status}" =~ "Running" ]] && [[ "${localkube_status}" =~ "Running" ]]; then
    return 0
  else
    return 1
  fi
}

function check_registry_access () {
  # check if the registry allows us to access the api, given our configured credentials

  if [[ $(echo "user = \"${DOCKER_USERNAME}:${DOCKER_PASSWORD}\"" | curl -K - -s "https://${DOCKER_SERVER}/v2/") == "{}" ]]; then
    log "verified access to $DOCKER_SERVER for $DOCKER_USERNAME"
  else
    die "registry access check failed.  check connectivity and credentials and try again."
  fi
}

function nat_allowance_for_xhyve () {
  if [[ $VM_DRIVER == "xhyve" ]]; then
    # list of all utun interfaces having ipv4 networks attached
    interfaces=( $(netstat -in | egrep 'utun\d .*\d+\.\d+\.\d+\.\d+' | cut -d ' ' -f 1 || true) )

    if [[ ${interfaces[@]} ]]; then
      rulefile="$(mktemp /tmp/minikube-utun-pfctl-nat-allowance.XXXXXX)"

      # flush the nat rules before modifying them
      sudo pfctl -a com.apple/tun -F nat

      for i in "${interfaces[@]}"; do
        RULE="nat on ${i} proto {tcp, udp, icmp} from 192.168.64.0/24 to any -> ${i}"
        echo $RULE >> $rulefile
      done

      log "adding pf rule allowing minikube traffic outbound via ${interfaces[@]}"
      sudo pfctl -a com.apple/tun -f $rulefile

      rm -f "$rulefile"
    else
      log "WARNING no utun interface found to configure.  If connecting to vpn past this point, minikube must be deleted and re-created."
    fi
  fi
}

# minikube mount can do this cleaner, but it is very slow for us.  so we prefer to use the mount provided by xhyve or virtualbox.
function minikube_link_builds () {
  GRANDPARENT_DIR_OF_SCRIPT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd -P )"
  WORK_REPO_DIR=${WORK_REPO_DIR:-$GRANDPARENT_DIR_OF_SCRIPT}

  # xhyve and virtualbox on osx mount at /Users, which works just fine inside and out of the guest.
  # but we need to refer to /home as /hosthome for virtualbox on linux.
  [[ $VM_DRIVER = "virtualbox" ]] && WORK_REPO_DIR=$(echo $WORK_REPO_DIR | sed -e 's/home/hosthome/g')
 
  log "linking /builds -> ${WORK_REPO_DIR} in minikube vm"
  minikube ssh "sudo ln -sf ${WORK_REPO_DIR} /builds"
}

# do all post-provisioning setup steps
function configure_minikube () {
  nat_allowance_for_xhyve

  log "configuring cluster dns"
  hack_kube_dns

  log "configuring registry access"
  setup_image_pull_secrets

  log "configuring /builds symlink"
  minikube_link_builds

  minikube_ip="$(minikube ip)"
  echo "Kubernetes node ($minikube_ip) successfully started and configured.  Huzzah!"
  echo
  echo "NOW WHAT:"
  echo "$ minikube dashboard      # open the kubernetes web app.  Note that the dashboard takes a moment to start up."
  echo "$ minikube ip             # get the current minikube IP; use this to access services running in minikube from your host."
  echo "$ kubectl get events -w   # watch state changes in the cluster for troubleshooting"
  echo "$ kubectl get pods        # show pods and their states"
  echo "$ minikube delete         # remove all minikube state and start over from scratch"
  echo "Deployment order suggested for bootstrapping the environment in minikube:"
  echo "TODO: this should be replaced with helm eventually"
  echo "$ bin/update_environment.sh -a backends"
  echo "$ bin/update_environment.sh -a stuff"
  echo "$ bin/update_environment.sh -a services"
  
}

function install_minikube_bin () {
  # if a dev build is requested, use that. otherwise, use the release
  if [[ $MINIKUBE_PR_NUMBER ]]; then
    MINIKUBE_BUILDS_ROOT="minikube-builds/$MINIKUBE_PR_NUMBER"
  elif [[ $MINIKUBE_VERSION ]]; then
    MINIKUBE_BUILDS_ROOT="minikube/releases/${MINIKUBE_VERSION}"
  else
    die "no minikube version information was requested."
    # we could implement 'latest-release' tracking here, but not sure if want
  fi

  case $OSTYPE in
    darwin*) MINIKUBE_ARCH_ARTIFACT=minikube-darwin-amd64  ;;
    linux*)  MINIKUBE_ARCH_ARTIFACT=minikube-linux-amd64   ;;
  esac

  MINIKUBE_BIN_URL="https://storage.googleapis.com/$MINIKUBE_BUILDS_ROOT/$MINIKUBE_ARCH_ARTIFACT"
  MINIKUBE_INSTALL_PREFIX="/usr/local/bin"

  if [[ $MINIKUBE_VERSION ]] && [[ ! $MINIKUBE_PR_NUMBER ]] ; then
    if [[ $($MINIKUBE_INSTALL_PREFIX/minikube version || true) =~ ${MINIKUBE_VERSION} ]]; then
      log "desired release $MINIKUBE_VERSION already installed [$($MINIKUBE_INSTALL_PREFIX/minikube version)]"
    else
      log "installed release [$($MINIKUBE_INSTALL_PREFIX/minikube version || true)] is not the desired release $MINIKUBE_VERSION"
      need_install=yes
    fi
  elif [[ $MINIKUBE_PR_NUMBER ]]; then
    log "desired version is a development build for pull request: https://github.com/kubernetes/minikube/pull/$MINIKUBE_PR_NUMBER"
    need_install=yes
  fi

  if [[ $need_install ]]; then
    log "installing $MINIKUBE_BIN_URL -> /usr/local/bin/minikube"
    curl --progress-bar -Lo /tmp/minikube $MINIKUBE_BIN_URL
    chmod +x /tmp/minikube
    sudo mv /tmp/minikube $MINIKUBE_INSTALL_PREFIX/minikube
  fi

}

function install_kubectl_bin () {
  # KUBECTL cluster management cli - http://kubernetes.io/docs/user-guide/prereqs/

  if [[ $(/usr/local/bin/kubectl version --client --short || true) =~ $KUBECTL_VERSION ]]; then
    log "kubectl $KUBECTL_VERSION already installed"
  else
    case $OSTYPE in
      darwin*) local kubectlbin=darwin/amd64  ;;
      linux*)  local kubectlbin=linux/amd64   ;;
    esac
    log "installing kubectl ${KUBECTL_VERSION} $kubectlbin"

    curl --progress-bar  -Lo /tmp/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${kubectlbin}/kubectl

    chmod +x /tmp/kubectl
    sudo mv /tmp/kubectl /usr/local/bin/
  fi
}

function install_xhyve_machine_driver () {

  # XHYVE https://github.com/zchee/docker-machine-driver-xhyve
  # virt backend with osx hooks; has working vpn dns whereas vbox doesn't
  # use of homebrew is nearly unavoidable for xhyve

  if docker-machine create --driver=xhyve --help >/dev/null; then
    log "docker-machine-driver-xhyve already installed"
  else
    log "installing docker-machine-driver-xhyve from homebrew"
    [[ -x "/usr/local/bin/brew" ]] || die "go install 'homebrew' [http://brew.sh/]"
    brew install docker-machine-driver-xhyve
  fi

  log "enabling setuid bit on docker-machine-driver-xhyve - this may prompt for sudo"
  sudo chown root:wheel /usr/local/opt/docker-machine-driver-xhyve/bin/docker-machine-driver-xhyve
  sudo chmod u+s /usr/local/opt/docker-machine-driver-xhyve/bin/docker-machine-driver-xhyve
}

function install_kvm_machine_driver () {
  if docker-machine create --driver=kvm --help >/dev/null; then
    log "docker-machine-driver-kvm already installed; to upgrade or reinstall, first run: rm /usr/local/bin/docker-machine{,-driver-kvm}"
  else
    [[ -f /etc/os-release ]] && source /etc/os-release

    local dm_md5sum="fd7b634f872885e91bc66fda6c39642e"
    curl -L https://github.com/docker/machine/releases/download/v0.11.0/docker-machine-Linux-x86_64 >/tmp/docker-machine
    [[ $(md5sum /tmp/docker-machine) =~ $dm_md5sum ]] || die "checksum mismatch: docker-machine"
    chmod +x /tmp/docker-machine
    sudo cp /tmp/docker-machine /usr/local/bin/docker-machine
    log "verified checksum, installed docker-machine"

    if [[ $ID == "ubuntu" ]] || [[ $ID == "debian" ]]; then
      local kvm_driver_release_md5sum="ece5cf96f817772091ebabfabada598c"
      local kvm_driver_release_url="https://github.com/dhiltgen/docker-machine-kvm/releases/download/v0.10.0/docker-machine-driver-kvm-ubuntu14.04"
    elif [[ $ID == "centos" ]] || [[ $ID == "fedora" ]]; then 
      local kvm_driver_release_md5sum="fb2ded7b5b20400ef66f0adbc384364e"
      local kvm_driver_release_url="https://github.com/dhiltgen/docker-machine-kvm/releases/download/v0.10.0/docker-machine-driver-kvm-centos7"
    fi

    curl -L $kvm_driver_release_url > /tmp/docker-machine-driver-kvm
    [[ $(md5sum /tmp/docker-machine-driver-kvm) =~ $kvm_driver_release_md5sum ]]
    chmod +x /tmp/docker-machine-driver-kvm
    sudo cp /tmp/docker-machine-driver-kvm /usr/local/bin/docker-machine-driver-kvm
    log "verified checksum, installed docker-machine-driver-kvm"
  fi
  log "LINUX-ONLY MANUAL STEP:  Install libvirt and qemu-kvm on your system (e.g., sudo apt-get install libvirt-bin qemu-kvm).  Add yourself to the libvirtd group (may vary by linux distro) so you don't need to sudo.  Run 'newgrp' after adding yourself or reboot before running this again."
}


function prompt_for_docker_credentials () {
  log "logging into registry [${DOCKER_SERVER}]"
  load_docker_credentials_if_present

  [[ $DOCKER_USERNAME ]] || read -p "${DOCKER_SERVER} username: " DOCKER_USERNAME
  [[ $DOCKER_PASSWORD ]] || read -r -s -p "${DOCKER_SERVER} password: " DOCKER_PASSWORD && echo

  test -n "$DOCKER_USERNAME" && test -n "$DOCKER_PASSWORD" && check_registry_access
}

function ensure_minikube_running () {
    if [[ $VM_DRIVER == "xhyve" ]]; then
      DRIVER="--vm-driver=xhyve"
      #VM_IP="192.168.64.1" # kinda
    elif [[ $VM_DRIVER == "virtualbox" ]]; then
      DRIVER="--vm-driver=virtualbox"
      #VM_IP="10.0.2.2" # sorta
    elif [[ $VM_DRIVER == "kvm" ]]; then
      DRIVER="--vm-driver=kvm"
      #VM_IP="192.168.42.1" # maybe
    fi

    if [[ $MINIKUBE_CPUS ]]; then
      local sys_cpus=$MINIKUBE_CPUS
    elif [[ $OSTYPE =~ darwin ]]; then
      local sys_cpus="$(sysctl -n hw.physicalcpu)"
    elif [[ $OSTYPE =~ linux ]]; then
      local sys_cpus="$(grep -c ^processor /proc/cpuinfo)"
    else
      local sys_cpus="1"
    fi

    local retries=3
    until minikube_is_running; do
      [[ $(( retries-- )) -eq 0 ]] && die "exceeded minikube setup retries"

      minikube start $DRIVER \
        --dns-domain ${DNS_DOMAIN:-custom.local} \
        --memory=${MINIKUBE_MEMORY:-4096} \
        --cpus=${sys_cpus}
      mk_err=$?

      # enables vpn dns resolution for virtualbox (not needed on xhyve)
      # for access to internal registries and http proxy
      # also not needed if using cisco anyconnect vpn client
      # probably not needed on linux, although I haven't tested this with vpnc or network-manager-vpnc
      if [[ $VM_DRIVER == "virtualbox" ]] \
      && [[ $OSTYPE =~ "darwin" ]] \
      && [[ $natdnshostresolver_setting_changed != "true" ]]; then
        log "restarting minikube vm to enable 'natdnshostresolver1'"
        minikube stop
        VBoxManage modifyvm minikube --natdnshostresolver1 on
        log "enabled natdnshostresolver on vbox vm [minikube]"
        natdnshostresolver_setting_changed=true
      fi
    done
}
# our registry requires a username and password, here, we attach
# that to the default service account as an imagePullSecret, after
# getting them from the user.

# http://kubernetes.io/docs/user-guide/service-accounts/#adding-imagepullsecrets-to-a-service-account
function setup_image_pull_secrets () {
  if kubectl get secret ${DOCKER_SERVER} &> /dev/null; then
    log "registry access secret [${DOCKER_SERVER}] already configured"
    log "if you want to change it, retry after running:  kubectl delete secret ${DOCKER_SERVER}"
    # to view: kubectl get secret ${DOCKER_SERVER} -o json | jq -r '.data.".dockercfg"'  | base64 -D | jq .
  else
    prompt_for_docker_credentials

    log "adding verified docker registry secret to kubernetes"
    kubectl create secret docker-registry ${DOCKER_SERVER} \
      --docker-server="${DOCKER_SERVER}" \
      --docker-username="${DOCKER_USERNAME}" \
      --docker-email="${DOCKER_USERNAME}@minikube" \
      --docker-password="${DOCKER_PASSWORD}"

    log "adding ${DOCKER_SERVER} secret to default serviceaccount"
    until kubectl get serviceaccount default &>/dev/null; do
      log "waiting for serviceaccount default"
      sleep 1
    done
    kubectl patch serviceaccount default -p "
imagePullSecrets:
  - name: ${DOCKER_SERVER}
"
  fi
}

function load_docker_credentials_if_present () {
  if [[ -r $HOME/.docker/config.json && -x $(which jq) ]]; then
    if [[ $(cat $HOME/.docker/config.json | jq -r .credsStore) =~ osxkeychain ]]; then
      local un=$(echo "$DOCKER_SERVER" | docker-credential-osxkeychain get | jq -r .Username)
      local pw=$(echo "$DOCKER_SERVER" | docker-credential-osxkeychain get | jq -r .Secret)
    else
      local authb64=$(jq -r '.auths."'${DOCKER_SERVER}'".auth' $HOME/.docker/config.json)
      local un=$(echo $authb64 | base64 --decode | cut -d: -f 1)
      local pw=$(echo $authb64 | base64 --decode | cut -d: -f 2)
    fi

    if [[ -n "$un" && -n "$pw" ]]; then
      log "got docker credentials from $HOME/.docker/config.json"
      export DOCKER_USERNAME="$un"
      export DOCKER_PASSWORD="$pw"
    fi
  fi
}

function hack_kube_dns () {
  log "replacing kube-dns for custom dns domain support"
  minikube addons disable kube-dns
  kubectl apply -f kube-dns.yaml
}


main