Ubuntu 18.04 developer environment configuration

big_data.sh

#!/bin/bash
# Spark
mkdir ~/spark && cd ~/spark && \
    wget http://apache.claz.org/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz -O spark.tgz && \
    tar zxvf spark.tgz --strip 1 && \
    rm spark.tgz
    
pip install pyspark
pip install kafka-python

# kafka
# https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-18-04
# Since Kafka can handle requests over a network, you should create a dedicated user for it.
# This minimizes damage to your Ubuntu machine should the Kafka server be compromised
## Creating a User for Kafka
sudo useradd kafka -m
sudo passwd kafka
sudo adduser kafka sudo
su -l kafka

## Downloading and Extracting the Kafka Binaries
mkdir ~/kafka && cd ~/kafka && \
    wget "http://www-eu.apache.org/dist/kafka/1.1.0/kafka_2.12-1.1.0.tgz" -O kafka.tgz && \
    tar zxvf kafka.tgz --strip 1 && \
    rm kafka.tgz

## Configuring the Kafka Server
# Kafka's default behavior will not allow us to delete a topic, the category, group, or feed name to which messages can be published
# ```echo "delete.topic.enable=true" >> ~/kafka/config/server.properties``` in case this append to the end of line but not a new line
echo -e "\ndelete.topic.enable=true" >> ~/kafka/config/server.properties

## Creating Systemd Unit Files and Starting the Kafka Server
# https://stackoverflow.com/questions/10134901/why-sudo-cat-gives-a-permission-denied-but-sudo-vim-works-fine
### Create the unit file for zookeeper
sudo bash -c 'cat>>/etc/systemd/system/zookeeper.service' <<EOF
[Unit]
Requires=network.target remote-fs.target
After=network.target remote-fs.target

[Service]
Type=simple
User=kafka
ExecStart=/home/kafka/kafka/bin/zookeeper-server-start.sh /home/kafka/kafka/config/zookeeper.properties
ExecStop=/home/kafka/kafka/bin/zookeeper-server-stop.sh
Restart=on-abnormal

[Install]
WantedBy=multi-user.target
EOF

### create the systemd service file for kafka:
sudo tee -a /etc/systemd/system/kafka.service >/dev/null <<EOF
[Unit]
Requires=zookeeper.service
After=zookeeper.service

[Service]
Type=simple
User=kafka
ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties > /home/kafka/kafka/kafka.log 2>&1'
ExecStop=/home/kafka/kafka/bin/kafka-server-stop.sh
Restart=on-abnormal

[Install]
WantedBy=multi-user.target
EOF

### start Kafka
sudo systemctl start kafka

# To ensure that the server has started successfully, check the journal logs for the kafka unit:
# user kafka cannot see the journal logs, since it is not in groups 'adm', 'systemd-journal'
# Hint: You are currently not seeing messages from other users and the system.
#       Users in groups 'adm', 'systemd-journal' can see all messages.
#       Pass -q to turn off this notice.
### so run this on your own user
journalctl -u kafka

### To enable kafka on server boot
sudo systemctl enable kafka

## Testing the Installation
### create a topic named test
~/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
bin/kafka-topics.sh --list --zookeeper localhost:2181
 
### Run the producer to send some messages
echo "Hello, World" | ~/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test > /dev/null

### Start a consumer in another terminal
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning

### Remove test topic
~/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --delete --topic test

# Restricting the Kafka User

# Remove the kafka user from the sudo group
sudo deluser kafka sudo

# lock the kafka user's password
sudo passwd kafka -l

# At this point, only root or a sudo user can log in as kafka by typing in the following command:
# sudo su - kafka

# unlock
# sudo passwd kafka -u

containerization&virtualization.sh

#!/bin/bash
# Virtualbox
wget https://download.virtualbox.org/virtualbox/5.2.16/virtualbox-5.2_5.2.16-123759~Ubuntu~bionic_amd64.deb -O virtualbox.deb && \
yes | sudo gdebi virtualbox.deb && \
rm virtualbox.deb

# Docker
## SET UP THE REPOSITORY
sudo apt-get update && \
    sudo apt install -y \
    apt-transport-https \
    ca-certificates \
    curl \
    software-properties-common

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository \
   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
   $(lsb_release -cs) \
   stable"
   
## INSTALL DOCKER CE
sudo apt-get update
sudo apt-get install -y docker-ce
sudo docker run hello-world

# Docker compose
sudo curl -L https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose

# autocompletion
# zsh
mkdir -p ~/.zsh/completion
curl -L https://raw.githubusercontent.com/docker/compose/1.22.0/contrib/completion/zsh/_docker-compose > ~/.zsh/completion/_docker-compose
fpath=(~/.zsh/completion $fpath)

# Kubectl
sudo snap install kubectl --classic
sudo snap install helm
sudo mv helm kubectl /usr/local/bin

# minikube
curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.28.2/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/

# kompose
curl -L https://github.com/kubernetes/kompose/releases/download/v1.1.0/kompose-linux-amd64 -o kompose
sudo mv kompose /usr/local/bin/

devOps.sh

#!/bin/bash
# git pre-installed
# sudo apt install -y git

# https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup
# git config --global user.name "Y** Y***"
# git config --global user.email "sh***@gmail.com"
# git config --global core.editor vim

# https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/
# ssh-keygen -t rsa -b 4096 -C "sh***@gmail.com"
# https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/
# copy paste ssh public key to github account (default: id_rsa.pub)
# Ansible
sudo apt-get update && \
    sudo apt-get install software-properties-common && \
    sudo apt-add-repository ppa:ansible/ansible && \
    sudo apt-get update && \
    sudo apt-get install ansible

# terraform 
wget https://releases.hashicorp.com/terraform/0.11.8/terraform_0.11.8_linux_amd64.zip -O terraform.zip
unzip terraform.zip
mv terraform /usr/local/bin/
rm terraform.zip

# GCP-SDK
# https://googlecloudplatform.github.io/google-cloud-python/latest/releases.html

# The google-cloud package is deprecated

# The google-cloud package is no longer maintained or updated. Instead, install the google-cloud-* subpackages directly.
pip install --upgrade google-cloud-storage
pip install --upgrade google-cloud-bigquery

# cli
export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
    echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - && \
    sudo apt-get update && sudo apt-get install google-cloud-sdk

gcloud init

frontend.sh

#!/bin/bash
# node, npm
sudo apt install -y \
    nodejs \
    npm
# node --version
# npm --version

# yarn
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - && \
    echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list && \
    sudo apt-get update && sudo apt-get install yarn
# yarn --version

# Dash
pip install dash==0.22.0  # The core dash backend
pip install dash-renderer==0.13.0  # The dash front-end
pip install dash-html-components==0.11.0  # HTML components
pip install dash-core-components==0.26.0  # Supercharged components
pip install plotly --upgrade  # Plotly graphing library used in examples

# npm global packages
sudo npm install -g create-react-app
sudo npm install -g hexo-cli 
sudo npm install -g http-server 

sudo apt install -y curl 
sudo apt install -y jq

pip install gunicorn
sudo snap install postman

nvidia_cuda_cudnn.sh

#!/bin/bash

# nvidia driver
sudo add-apt-repository ppa:graphics-drivers/ppa
sudo apt-get update
sudo ubuntu-drivers autoinstall
# nvidia-smi should show related information after reboot, otherwise there would be error
# NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver
# nvidia-smi

# cuda
# https://medium.com/@asmello/how-to-install-tensorflow-cuda-9-1-into-ubuntu-18-04-b645e769f01d
sudo apt install nvidia-cuda-toolkit
sudo apt install libcupti-dev
nvcc --version
# cuda 9.1

# apt would install cuda related packages into root system paths: /usr/bin, /usr/include,/usr/lib
# while packages like tensorflow would look for them in /usr/local/cuda-*.*
# so some symbolic links would be helpful
sudo mkdir -p /usr/local/cuda /usr/local/cuda/extras/CUPTI /usr/local/cuda/nvvm
sudo ln -s /usr/bin /usr/local/cuda/bin
sudo ln -s /usr/include /usr/local/cuda/include
sudo ln -s /usr/lib/x86_64-linux-gnu /usr/local/cuda/lib64
sudo ln -s /usr/local/cuda/lib64 /usr/local/cuda/lib
sudo ln -s /usr/include /usr/local/cuda/extras/CUPTI/include
sudo ln -s /usr/lib/x86_64-linux-gnu /usr/local/cuda/extras/CUPTI/lib64
sudo ln -s /usr/lib/nvidia-cuda-toolkit/libdevice /usr/local/cuda/nvvm/libdevice

# cuDNN
# https://developer.nvidia.com/cudnn
# sign in or log in to your Nvidia account here

# https://developer.nvidia.com/rdp/cudnn-archive
# Download according to cuda version
# Download cuDNN v7.0.5 (Dec 11, 2017), for CUDA 9.1 >> cuDNN v7.0.5 Library for Linux
sudo tar zxvf ~/Downloads/cudnn-9.1-linux-x64-v7.tgz -C /usr/local

python_machine_learning.sh

#!/bin/bash
# Anaconda, Python
wget --quiet https://repo.anaconda.com/archive/Anaconda3-5.2.0-Linux-x86_64.sh -O ~/anaconda.sh
# accept the license terms and install to ~/anaconda but have to add env path manually
# /bin/bash ~/anaconda.sh -b -p ~/anaconda
# echo 'export PATH="~/anaconda/bin:$PATH"' >> ~/.bashrc
/bin/bash ~/anaconda.sh
rm ~/anaconda.sh
source ~/.bashrc

# some python packages
pip install --upgrade pip
pip install msgpack
pip install autopep8
pip install pycodestyle
pip install tqdm
pip install google-api-python-client
pip install requests
pip install kafka-python
pip install tweepy
pip install pyyaml

# Beakerx
# require nodejs, npm, yarn
# However, nodejs 10 would make yarn hangs during installation, nodejs 8 is fine
# [1/4] Resolving packages...
# https://github.com/jupyter-widgets/ipywidgets/issues/2061
conda update -n base conda && \
    conda config --env --add pinned_packages 'openjdk>8.0.121' && \
    jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
    jupyter labextension install beakerx-jupyterlab
    
# Tensorflow
# https://www.tensorflow.org/install/install_sources
# Build from source since ```pip install tensorflow-gpu``` would use default cuda 9.0 with error like
# ImportError: libcublas.so.9.0: cannot open shared object file: No such file or directory
git clone https://github.com/tensorflow/tensorflow
cd tensorflow

# bazel, official tool to build tensorflow
sudo apt install openjdk-8-jdk
echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add -
sudo apt update && sudo apt install bazel

./configure
# Noteworthy points:
## https://docs.nvidia.com/cuda/archive/9.1/cuda-installation-guide-linux/index.html
## cuda 9.1 does not support gcc-7
# 1. GCC path: /usr/bin/gcc-6 
# 2. cuda version: 9.1
# 3. cuDNN version: 7.0.5
## NCCL is used for multiple GPU, it is fine to use 1.3 if there is only one GPU since 1.3 would be automatically fetched
## otherwise NCCL version 2 should be better but it need a bit more work to be installed beforehand
# 4. NCCL version: 1.3
## https://developer.nvidia.com/cuda-gpus
# 5. Cuda compute capabilities

bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package
bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
pip install /tmp/tensorflow_pkg/tensorflow*.whl

# Pytorch
conda install pytorch torchvision cuda91 -c pytorch

# Keras
pip install keras

# Xgboost
# https://xgboost.readthedocs.io/en/latest/build.html
# unsupported GNU version! gcc versions later than 6 are not supported!
# https://github.com/dmlc/xgboost/issues/2787
# https://stackoverflow.com/questions/17275348/how-to-specify-new-gcc-path-for-cmake
# export CC=/usr/local/bin/gcc-6
cd ~ && \
    git clone --recursive https://github.com/dmlc/xgboost && \
    cd xgboost && mkdir build && cd build && cmake -DCUDA_HOST_COMPILER=/usr/bin/gcc-6 -DUSE_CUDA=ON .. && make -j && \
    cd ../python-package && python setup.py install

# LightGBM
# https://lightgbm.readthedocs.io/en/latest/GPU-Tutorial.html
sudo apt install -y \
    libboost-dev \
    libboost-system-dev \
    libboost-filesystem-dev

cd ~ && \
    git clone --recursive https://github.com/Microsoft/LightGBM && \
    cd LightGBM && mkdir build && cd build && \
    cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ .. && \
    make -j && \
    cd ../python-package && python setup.py install --precompile

# NLP
pip install -U nltk
# https://www.nltk.org/data.html
# default location: /usr/share/nltk_data
# use -d flag to specify a different location
python -m nltk.downloader all

pip install Unidecode
pip install jieba
pip install wordcloud
pip install vaderSentiment

Ubuntu18.04_general_configuration.sh

#!/bin/bash
sudo apt-get update --fix-missing

# gdebi replace dpkg to solve dependencies during installation
sudo apt install -y gdebi
sudo apt install -y cmake

# communication 
# Slack -- ubuntu software

# editor
## vim
sudo apt install -y vim

## zsh
sudo apt-get install zsh
sh -c "$(wget https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh -O -)"

## atom 
wget https://atom.io/download/deb -O atom.deb && \
yes | sudo gdebi atom.deb && \
rm atom.deb

# image processing
## image editor
sudo apt install gimp
sudo apt install pinta

## screenshot
sudo apt install flameshot

# gnome editor
sudo apt install dconf-editor

# browser
sudo apt install -y torbrowser-launcher 
sudo apt install -y chromium-browser
  
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -O chrome.deb && \
yes | sudo gdebi chrome.deb && \
rm chrome.deb

# dropbox
wget https://www.dropbox.com/download?dl=packages/ubuntu/dropbox_2015.10.28_amd64.deb -O dropbox.deb && \
yes | sudo gdebi dropbox.deb && \
rm dropbox.deb

# music player
# wget https://github.com/trazyn/ieaseMusic/releases/download/v1.2.9/ieaseMusic-1.2.9-linux-amd64.deb -O ieaseMusic.deb && \
# yes | sudo gdebi ieaseMusic.deb && \
# rm ieaseMusic.deb

wget http://d1.music.126.net/dmusic/netease-cloud-music_1.1.0_amd64_ubuntu.deb -O netease-cloud-music.deb && \
yes | sudo gdebi netease-cloud-music.deb && \
rm netease-cloud-music.deb
# https://github.com/MajeurAndroid/Adb-Remote-Screen/issues/14
# Failed to load module "canberra-gtk-module" 
sudo apt install -y libcanberra-gtk-module
# Local file: "" ("netease-cloud-music")
# Suspend from both command line and desktop -> ```sudo netease-cloud-music``` or click shutdown then cancel

# input
sudo apt install -y \
    fcitx-bin \
    fcitx-googlepinyin

# use " since there is characters &
wget "http://cdn2.ime.sogou.com/dl/index/1524572264/sogoupinyin_2.2.0.0108_amd64.deb?st=qC_O2p5443g1a2TJR_rSdA&e=1533163019&fn=sogoupinyin_2.2.0.0108_amd64.deb" -O sogoupinyin.deb && \
    yes | sudo gdebi sogoupinyin.deb && \
    rm sogoupinyin.deb
    
# reboot -> fcitx configure -> add input

# VPN
## ShadowSocks
pip install git+https://github.com/shadowsocks/shadowsocks.git@master

## openvpn
sudo apt install -y \ 
    openvpn \
    network-manager-openvpn \
    network-manager-openvpn-gnome

# print directory structure in the form of a tree
sudo apt-get install tree
sudo apt-get install -y openssh-server
sudo apt install net-tools # ifconfig
sudo apt-get install -y nmap
sudo apt-get install -y fail2ban
cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local

# latex
sudo apt install -y texlive-full

# 7zip
sudo apt-get install -y p7zip-full p7zip-rar