Skip to content

Instantly share code, notes, and snippets.

@rafaeltuelho

rafaeltuelho/osev3-basic-installation-steps.md

Last active May 4, 2018 14:19
Show Gist options
  • Save rafaeltuelho/3ca43a226b538fc230dccb5b7ba7d933 to your computer and use it in GitHub Desktop.
Save rafaeltuelho/3ca43a226b538fc230dccb5b7ba7d933 to your computer and use it in GitHub Desktop.
Download ZIP
OSEv3 Basic installation steps (commands)
Raw
osev3-basic-installation-steps.md

Environment Setup

SSH Access

ssh rsoares-redhat.com@oselab-dcff.oslab.opentlc.com 
	ssh infranode00-dcff
	ssh master00-dcff
	ssh node00-dcff
	ssh node01-dcff

ssh-keygen -f /root/.ssh/id_rsa -N ''
cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
echo StrictHostKeyChecking no >> /etc/ssh/ssh_config
ssh master00-$guid "echo StrictHostKeyChecking no >> /etc/ssh/ssh_config"
ssh-copy-id master00-$GUID.oslab.opentlc.com

installing ssh-key on cluster members

for node in \
	master00-$GUID.oslab.opentlc.com infranode00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
   ssh-copy-id root@$node ; \
done

configuring yum repos

cat << EOF > /etc/yum.repos.d/open.repo
[rhel-x86_64-server-7]
name=Red Hat Enterprise Linux 7
baseurl=http://www.opentlc.com/repos/ose/3.1/rhel-7-server-rpms
enabled=1
gpgcheck=0
[rhel-x86_64-server-extras-7]
name=Red Hat Enterprise Linux 7 Extras
baseurl=http://www.opentlc.com/repos/ose/3.1/rhel-7-server-extras-rpms
enabled=1
gpgcheck=0
[rhel-x86_64-server-optional-7]
name=Red Hat Enterprise Linux 7 Optional
baseurl=http://www.opentlc.com/repos/ose/3.1/rhel-7-server-optional-rpms
enabled=1
gpgcheck=0
# This repo is added for the OPENTLC environment not OSE
[rhel-x86_64-server-rh-common-7]
name=Red Hat Enterprise Linux 7 Commonbaseurl=http://www.opentlc.com/repos/ose/3.1/rhel-7-server-rh-common-rpms
enabled=1
gpgcheck=0
EOF

cat << EOF >> /etc/yum.repos.d/open.repo
[rhel-7-server-ose-3.1-rpms]name=Red Hat Enterprise Linux 7 OSE 3.1
baseurl=http://www.opentlc.com/repos/ose/3.1/rhel-7-server-ose-3.1-rpms
enabled=1
gpgcheck=0
EOF

for node in \
	master00-$GUID.oslab.opentlc.com infranode00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
   echo Copying open repos to $node; \
   scp /etc/yum.repos.d/open.repo ${node}:/etc/yum.repos.d/open.repo; \
   yum clean all; yum repolist; \
done

for node in \
	master00-$GUID.oslab.opentlc.com infranode00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
   echo removing NetworkManager on $node; \
   ssh $node "yum -y  remove NetworkManager*"; \
done

ssh master00-$guid "yum -y install wget git net-tools bind-utils iptables-services bridge-utils python-virtualenv gcc"

yum -y install "bash-completion"
ssh master00-$guid "yum -y install bash-completion"

for node in \
	master00-$GUID.oslab.opentlc.com infranode00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
   echo updating pkgs on $node; \
   ssh $node "yum -y update"; \
done

Docker

ssh master00-$GUID

yum -y install docker

sed -i "s|OPTIONS.*|OPTIONS='--selinux-enabled --insecure-registry 172.30.0.0/16'|" /etc/sysconfig/docker

for node in \
	infranode00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
	echo installing Docker on $node; \
	ssh $node "yum -y install docker"; \
done

for node in \
	infranode00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
	echo Overwriting docker configuration file on $node; \
	scp  /etc/sysconfig/docker $node:/etc/sysconfig/docker; \
done

for node in \
	infranode00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
	echo Cleaning up Docker file on $node; \
	ssh $node "systemctl stop docker ; rm -rf /var/lib/docker/*"; \
done

cat <<EOF > /etc/sysconfig/docker-storage-setup
DEVS=/dev/vdb
VG=docker-vg
EOF

docker-storage-setup

for node in \
   infranode00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
   echo Configuring Docker Storage and rebooting $node; \
   scp /etc/sysconfig/docker-storage-setup ${node}:/etc/sysconfig/docker-storage-setup; \
   ssh $node "docker-storage-setup; systemctl enable docker;reboot"; \
done

reboot

Pull Docker images on cluster's members

on oselab host...

REGISTRY="registry.access.redhat.com";PTH="openshift3"

for node in \
   node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
   ssh $node "docker pull $REGISTRY/$PTH/ose-deployer:v3.1.0.4; docker pull $REGISTRY/$PTH/ose-sti-builder:v3.1.0.4; docker pull $REGISTRY/$PTH/ose-pod:v3.1.0.4; docker pull $REGISTRY/$PTH/ose-keepalived-ipfailover:v3.1.0.4; docker pull $REGISTRY/$PTH/ruby-20-rhel7; docker pull $REGISTRY/$PTH/mysql-55-rhel7; docker pull openshift/hello-openshift:v1.0.6;" \
done

node=infranode00-$guid.oslab.opentlc.com

ssh $node "docker pull $REGISTRY/$PTH/ose-haproxy-router:v3.1.0.4  ; docker pull $REGISTRY/$PTH/ose-deployer:v3.1.0.4 ; docker pull $REGISTRY/$PTH/ose-pod:v3.1.0.4 ; docker pull $REGISTRY/$PTH/ose-docker-registry:v3.1.0.4 ;"

yum -y install atomic-openshift-utils
atomic-openshift-installer install

for node in \
   infranode00-$guid.oslab.opentlc.com master00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
   echo rebooting $node; \
   ssh $node "reboot"; \
done

ssh master00-$guid

oc get nodes

Configure and Setup OpenShift Enterprise.

on master00 host...

oc label node infranode00-$GUID.oslab.opentlc.com region="infra" zone="infranodes"
oc label node node00-$GUID.oslab.opentlc.com region="primary" zone="east"
oc label node node01-$GUID.oslab.opentlc.com region="primary" zone="west"

oc get nodes

cp /etc/origin/master/master-config.yaml /etc/origin/master/master-config.yaml.original

sed  -i "s|subdomain:  \"\"|subdomain: \"cloudapps-${GUID}.oslab.opentlc.com\"|g" /etc/origin/master/master-config.yaml
systemctl restart atomic-openshift-master

sed -i 's|defaultNodeSelector: \"\"|defaultNodeSelector: \"region=primary\"|' /etc/origin/master/master-config.yaml

systemctl restart atomic-openshift-master
systemctl status atomic-openshift-master

oc edit namespace default

apiVersion: v1
kind: Namespace
metadata:
  annotations:
    openshift.io/node-selector: region=infra
...

oc get namespace default -o yaml

to see service's log

journalctl -f -u atomic-openshift-master
journalctl -f -u atomic-openshift-node
journalctl -f -u docker

Authentication

yum -y install httpd-tools
htpasswd -cb /etc/origin/openshift-passwd tuelho r3dh4t1!

vim 

oauthConfig:  
   assetPublicURL: https://master00-GUID.oslab.opentlc.com:8443/console/  grantConfig:
       method: auto
         identityProviders:
           - name: htpasswd_auth
               challenge: true
               login: true    
               provider:
                     apiVersion: v1
                     kind: HTPasswdPasswordIdentityProvider      
                     file: /etc/origin/openshift-passwd  

systemctl restart atomic-openshift-master

Registry and Router

oadm registry --create --credentials=/etc/origin/master/openshift-registry.kubeconfig

oc get pods
NAME                      READY     STATUS    RESTARTS   AGE
docker-registry-1-4b628   1/1       Running   0          1m

curl -v 172.30.213.79:5000/healthz

curl -v `oc get service docker-registry --template '{{.spec.portalIP}}:{{index .spec.ports 0 "port"}}/healthz'`

CA=/etc/origin/master
oadm ca create-server-cert --signer-cert=$CA/ca.crt --signer-key=$CA/ca.key --signer-serial=$CA/ca.serial.txt --hostnames='*.cloudapps-$guid.oslab.opentlc.com' --cert=cloudapps.crt --key=cloudapps.key

cat cloudapps.crt cloudapps.key $CA/ca.crt > /etc/origin/master/cloudapps.router.pem

oadm router trainingrouter --replicas=1 --credentials='/etc/origin/master/openshift-router.kubeconfig' --service-account=router --stats-password='r3dh@t1!'

oc get pods -w

Populate OpenShift Enterprise

oc create|delete -f /usr/share/openshift/examples/image-streams/image-streams-rhel7.json -n openshift

oc create|delete or remove -f /usr/share/openshift/examples/db-templates -n openshift

oc create|delete -f /usr/share/openshift/examples/quickstart-templates -n openshift

Configure NFS Persistent Storage

for node in \
   infranode00-$guid.oslab.opentlc.com node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
   echo Installing nfs-utils on $node; \
   ssh $node "yum install -y nfs-utils"; \
done

mkdir -p /var/export/pvs/pv{1..100}
chown -R nfsnobody:nfsnobody /var/export/pvs/
chmod -R 700 /var/export/pvs/

for volume in pv{1..100}; \
do \
   echo Creating export for volume $volume; \
   echo "/var/export/pvs/${volume} 192.168.0.0/24(rw,sync,all_squash)" >> /etc/exports; \
done;

iptables -I INPUT 1 -p tcp --dport 2049 -j ACCEPT
iptables -I INPUT 1 -p tcp --dport 20048 -j ACCEPT
iptables -I INPUT 1 -p tcp --dport 111 -j ACCEPT
service iptables save

systemctl enable rpcbind nfs-server
systemctl start rpcbind nfs-server nfs-lock nfs-idmap
systemctl restart iptables

for node in \
   node00-$guid.oslab.opentlc.com node01-$guid.oslab.opentlc.com; \
do \
	echo Setting SElinux Policy on $node; \
	ssh $node "setsebool -P virt_use_nfs=true"; \
done

ssh 192.168.0.20x
mkdir /tmp/test
mount -v 192.168.0.254:/var/export/pvs/pv98 /tmp/test
# Check if any errors accure and unmount.[root@node0X-GUID ~]# umount /tmp/testexit

Create Definition Files for Volumes

ssh master00-$guid
mkdir /root/pvs

export volsize="5Gi"
for volume in pv{1..25}; \
do \
cat << EOF > /root/pvs/${volume}.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: ${volume} 
spec:
  capacity:
    storage: ${volsize} 
  accessModes:
  - ReadWriteOnce 
  nfs: 
    path: /var/export/pvs/${volume} 
    server: 192.168.0.254 
  persistentVolumeReclaimPolicy: Recycle 
EOF
     echo "Created def file for ${volume}"; \
done

export volsize="10Gi"
for volume in pv{26..50}; \
do \
cat << EOF > /root/pvs/${volume}.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: ${volume} 
spec:
  capacity:
    storage: ${volsize} 
  accessModes:
  - ReadWriteOnce 
  nfs: 
    path: /var/export/pvs/${volume} 
    server: 192.168.0.254 
  persistentVolumeReclaimPolicy: Recycle 
EOF
     echo "Created def file for ${volume}"; \
done

export volsize="1Gi"
for volume in pv{51..100}; \
do \
cat << EOF > /root/pvs/${volume}.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: ${volume} 
spec:
  capacity:
    storage: ${volsize} 
  accessModes:
  - ReadWriteOnce 
  nfs: 
    path: /var/export/pvs/${volume} 
    server: 192.168.0.254 
  persistentVolumeReclaimPolicy: Recycle 
EOF
     echo "Created def file for ${volume}"; \
done

cd /root/pvs
oc create -f /root/pv21.yaml -n default
oc create -f /root/pv22.yaml -n default
oc create -f /root/pv23.yaml -n default

creating a new NFS export to be used in Registry

[root@oselab-dcff ~]# export volname=registry-storage
[root@oselab-dcff ~]# mkdir -p /var/export/pvs/${volname}
[root@oselab-dcff ~]# chown nfsnobody:nfsnobody /var/export/pvs/${volname}
[root@oselab-dcff ~]# chmod 700 /var/export/pvs/${volname}
[root@oselab-dcff ~]# echo "/var/export/pvs/${volname} *(rw,sync,all_squash)" >> /etc/exports
[root@oselab-dcff ~]# systemctl restart rpcbind nfs-server nfs-lock nfs-idmap

As root on the master host, create a persistent volume-definition file named registry-volume.json:

[root@oselab-GUID ~]# ssh master00-$guid
[root@master00-GUID ~]# cat << EOF > registry-volume.json
    {
      "apiVersion": "v1",
      "kind": "PersistentVolume",
      "metadata": {
        "name": "registry-storage"
      },
      "spec": {
        "capacity": {
            "storage": "15Gi"
            },
        "accessModes": [ "ReadWriteMany" ],
        "nfs": {
            "path": "/var/export/pvs/registry-storage",
            "server": "oselab-${GUID}.oslab.opentlc.com"
        }
      }
    }

EOF

oc create -f registry-volume.json -n default
oc get pv

cat << EOF > registry-volume-claim.json
    {
      "apiVersion": "v1",
      "kind": "PersistentVolumeClaim",
      "metadata": {
        "name": "registry-claim"
      },
      "spec": {
        "accessModes": [ "ReadWriteMany" ],
        "resources": {
          "requests": {
            "storage": "15Gi"
          }
        }
      }
    }

EOF

[root@master00-dcff ~]# oc create -f registry-volume-claim.json -n default
oc get pvc

to attach the PV...

[root@master00-dcff ~]# oc volume dc/docker-registry --add --overwrite -t persistentVolumeClaim --claim-name=registry-claim --name=registry-storage
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment