Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Ubuntu 18.04 developer environment configuration
#!/bin/bash
# Spark
mkdir ~/spark && cd ~/spark && \
wget http://apache.claz.org/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz -O spark.tgz && \
tar zxvf spark.tgz --strip 1 && \
rm spark.tgz
pip install pyspark
pip install kafka-python
# kafka
# https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-18-04
# Since Kafka can handle requests over a network, you should create a dedicated user for it.
# This minimizes damage to your Ubuntu machine should the Kafka server be compromised
## Creating a User for Kafka
sudo useradd kafka -m
sudo passwd kafka
sudo adduser kafka sudo
su -l kafka
## Downloading and Extracting the Kafka Binaries
mkdir ~/kafka && cd ~/kafka && \
wget "http://www-eu.apache.org/dist/kafka/1.1.0/kafka_2.12-1.1.0.tgz" -O kafka.tgz && \
tar zxvf kafka.tgz --strip 1 && \
rm kafka.tgz
## Configuring the Kafka Server
# Kafka's default behavior will not allow us to delete a topic, the category, group, or feed name to which messages can be published
# ```echo "delete.topic.enable=true" >> ~/kafka/config/server.properties``` in case this append to the end of line but not a new line
echo -e "\ndelete.topic.enable=true" >> ~/kafka/config/server.properties
## Creating Systemd Unit Files and Starting the Kafka Server
# https://stackoverflow.com/questions/10134901/why-sudo-cat-gives-a-permission-denied-but-sudo-vim-works-fine
### Create the unit file for zookeeper
sudo bash -c 'cat>>/etc/systemd/system/zookeeper.service' <<EOF
[Unit]
Requires=network.target remote-fs.target
After=network.target remote-fs.target
[Service]
Type=simple
User=kafka
ExecStart=/home/kafka/kafka/bin/zookeeper-server-start.sh /home/kafka/kafka/config/zookeeper.properties
ExecStop=/home/kafka/kafka/bin/zookeeper-server-stop.sh
Restart=on-abnormal
[Install]
WantedBy=multi-user.target
EOF
### create the systemd service file for kafka:
sudo tee -a /etc/systemd/system/kafka.service >/dev/null <<EOF
[Unit]
Requires=zookeeper.service
After=zookeeper.service
[Service]
Type=simple
User=kafka
ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties > /home/kafka/kafka/kafka.log 2>&1'
ExecStop=/home/kafka/kafka/bin/kafka-server-stop.sh
Restart=on-abnormal
[Install]
WantedBy=multi-user.target
EOF
### start Kafka
sudo systemctl start kafka
# To ensure that the server has started successfully, check the journal logs for the kafka unit:
# user kafka cannot see the journal logs, since it is not in groups 'adm', 'systemd-journal'
# Hint: You are currently not seeing messages from other users and the system.
# Users in groups 'adm', 'systemd-journal' can see all messages.
# Pass -q to turn off this notice.
### so run this on your own user
journalctl -u kafka
### To enable kafka on server boot
sudo systemctl enable kafka
## Testing the Installation
### create a topic named test
~/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
bin/kafka-topics.sh --list --zookeeper localhost:2181
### Run the producer to send some messages
echo "Hello, World" | ~/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test > /dev/null
### Start a consumer in another terminal
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
### Remove test topic
~/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --delete --topic test
# Restricting the Kafka User
# Remove the kafka user from the sudo group
sudo deluser kafka sudo
# lock the kafka user's password
sudo passwd kafka -l
# At this point, only root or a sudo user can log in as kafka by typing in the following command:
# sudo su - kafka
# unlock
# sudo passwd kafka -u
#!/bin/bash
# Virtualbox
wget https://download.virtualbox.org/virtualbox/5.2.16/virtualbox-5.2_5.2.16-123759~Ubuntu~bionic_amd64.deb -O virtualbox.deb && \
yes | sudo gdebi virtualbox.deb && \
rm virtualbox.deb
# Docker
## SET UP THE REPOSITORY
sudo apt-get update && \
sudo apt install -y \
apt-transport-https \
ca-certificates \
curl \
software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
## INSTALL DOCKER CE
sudo apt-get update
sudo apt-get install -y docker-ce
sudo docker run hello-world
# Docker compose
sudo curl -L https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
# autocompletion
# zsh
mkdir -p ~/.zsh/completion
curl -L https://raw.githubusercontent.com/docker/compose/1.22.0/contrib/completion/zsh/_docker-compose > ~/.zsh/completion/_docker-compose
fpath=(~/.zsh/completion $fpath)
# Kubectl
sudo snap install kubectl --classic
sudo snap install helm
sudo mv helm kubectl /usr/local/bin
# minikube
curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.28.2/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
# kompose
curl -L https://github.com/kubernetes/kompose/releases/download/v1.1.0/kompose-linux-amd64 -o kompose
sudo mv kompose /usr/local/bin/
#!/bin/bash
# git pre-installed
# sudo apt install -y git
# https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup
# git config --global user.name "Y** Y***"
# git config --global user.email "sh***@gmail.com"
# git config --global core.editor vim
# https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/
# ssh-keygen -t rsa -b 4096 -C "sh***@gmail.com"
# https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/
# copy paste ssh public key to github account (default: id_rsa.pub)
# Ansible
sudo apt-get update && \
sudo apt-get install software-properties-common && \
sudo apt-add-repository ppa:ansible/ansible && \
sudo apt-get update && \
sudo apt-get install ansible
# terraform
wget https://releases.hashicorp.com/terraform/0.11.8/terraform_0.11.8_linux_amd64.zip -O terraform.zip
unzip terraform.zip
mv terraform /usr/local/bin/
rm terraform.zip
# GCP-SDK
# https://googlecloudplatform.github.io/google-cloud-python/latest/releases.html
# The google-cloud package is deprecated
# The google-cloud package is no longer maintained or updated. Instead, install the google-cloud-* subpackages directly.
pip install --upgrade google-cloud-storage
pip install --upgrade google-cloud-bigquery
# cli
export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - && \
sudo apt-get update && sudo apt-get install google-cloud-sdk
gcloud init
#!/bin/bash
# node, npm
sudo apt install -y \
nodejs \
npm
# node --version
# npm --version
# yarn
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - && \
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list && \
sudo apt-get update && sudo apt-get install yarn
# yarn --version
# Dash
pip install dash==0.22.0 # The core dash backend
pip install dash-renderer==0.13.0 # The dash front-end
pip install dash-html-components==0.11.0 # HTML components
pip install dash-core-components==0.26.0 # Supercharged components
pip install plotly --upgrade # Plotly graphing library used in examples
# npm global packages
sudo npm install -g create-react-app
sudo npm install -g hexo-cli
sudo npm install -g http-server
sudo apt install -y curl
sudo apt install -y jq
pip install gunicorn
sudo snap install postman
#!/bin/bash
# nvidia driver
sudo add-apt-repository ppa:graphics-drivers/ppa
sudo apt-get update
sudo ubuntu-drivers autoinstall
# nvidia-smi should show related information after reboot, otherwise there would be error
# NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver
# nvidia-smi
# cuda
# https://medium.com/@asmello/how-to-install-tensorflow-cuda-9-1-into-ubuntu-18-04-b645e769f01d
sudo apt install nvidia-cuda-toolkit
sudo apt install libcupti-dev
nvcc --version
# cuda 9.1
# apt would install cuda related packages into root system paths: /usr/bin, /usr/include,/usr/lib
# while packages like tensorflow would look for them in /usr/local/cuda-*.*
# so some symbolic links would be helpful
sudo mkdir -p /usr/local/cuda /usr/local/cuda/extras/CUPTI /usr/local/cuda/nvvm
sudo ln -s /usr/bin /usr/local/cuda/bin
sudo ln -s /usr/include /usr/local/cuda/include
sudo ln -s /usr/lib/x86_64-linux-gnu /usr/local/cuda/lib64
sudo ln -s /usr/local/cuda/lib64 /usr/local/cuda/lib
sudo ln -s /usr/include /usr/local/cuda/extras/CUPTI/include
sudo ln -s /usr/lib/x86_64-linux-gnu /usr/local/cuda/extras/CUPTI/lib64
sudo ln -s /usr/lib/nvidia-cuda-toolkit/libdevice /usr/local/cuda/nvvm/libdevice
# cuDNN
# https://developer.nvidia.com/cudnn
# sign in or log in to your Nvidia account here
# https://developer.nvidia.com/rdp/cudnn-archive
# Download according to cuda version
# Download cuDNN v7.0.5 (Dec 11, 2017), for CUDA 9.1 >> cuDNN v7.0.5 Library for Linux
sudo tar zxvf ~/Downloads/cudnn-9.1-linux-x64-v7.tgz -C /usr/local
#!/bin/bash
# Anaconda, Python
wget --quiet https://repo.anaconda.com/archive/Anaconda3-5.2.0-Linux-x86_64.sh -O ~/anaconda.sh
# accept the license terms and install to ~/anaconda but have to add env path manually
# /bin/bash ~/anaconda.sh -b -p ~/anaconda
# echo 'export PATH="~/anaconda/bin:$PATH"' >> ~/.bashrc
/bin/bash ~/anaconda.sh
rm ~/anaconda.sh
source ~/.bashrc
# some python packages
pip install --upgrade pip
pip install msgpack
pip install autopep8
pip install pycodestyle
pip install tqdm
pip install google-api-python-client
pip install requests
pip install kafka-python
pip install tweepy
pip install pyyaml
# Beakerx
# require nodejs, npm, yarn
# However, nodejs 10 would make yarn hangs during installation, nodejs 8 is fine
# [1/4] Resolving packages...
# https://github.com/jupyter-widgets/ipywidgets/issues/2061
conda update -n base conda && \
conda config --env --add pinned_packages 'openjdk>8.0.121' && \
jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
jupyter labextension install beakerx-jupyterlab
# Tensorflow
# https://www.tensorflow.org/install/install_sources
# Build from source since ```pip install tensorflow-gpu``` would use default cuda 9.0 with error like
# ImportError: libcublas.so.9.0: cannot open shared object file: No such file or directory
git clone https://github.com/tensorflow/tensorflow
cd tensorflow
# bazel, official tool to build tensorflow
sudo apt install openjdk-8-jdk
echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add -
sudo apt update && sudo apt install bazel
./configure
# Noteworthy points:
## https://docs.nvidia.com/cuda/archive/9.1/cuda-installation-guide-linux/index.html
## cuda 9.1 does not support gcc-7
# 1. GCC path: /usr/bin/gcc-6
# 2. cuda version: 9.1
# 3. cuDNN version: 7.0.5
## NCCL is used for multiple GPU, it is fine to use 1.3 if there is only one GPU since 1.3 would be automatically fetched
## otherwise NCCL version 2 should be better but it need a bit more work to be installed beforehand
# 4. NCCL version: 1.3
## https://developer.nvidia.com/cuda-gpus
# 5. Cuda compute capabilities
bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package
bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
pip install /tmp/tensorflow_pkg/tensorflow*.whl
# Pytorch
conda install pytorch torchvision cuda91 -c pytorch
# Keras
pip install keras
# Xgboost
# https://xgboost.readthedocs.io/en/latest/build.html
# unsupported GNU version! gcc versions later than 6 are not supported!
# https://github.com/dmlc/xgboost/issues/2787
# https://stackoverflow.com/questions/17275348/how-to-specify-new-gcc-path-for-cmake
# export CC=/usr/local/bin/gcc-6
cd ~ && \
git clone --recursive https://github.com/dmlc/xgboost && \
cd xgboost && mkdir build && cd build && cmake -DCUDA_HOST_COMPILER=/usr/bin/gcc-6 -DUSE_CUDA=ON .. && make -j && \
cd ../python-package && python setup.py install
# LightGBM
# https://lightgbm.readthedocs.io/en/latest/GPU-Tutorial.html
sudo apt install -y \
libboost-dev \
libboost-system-dev \
libboost-filesystem-dev
cd ~ && \
git clone --recursive https://github.com/Microsoft/LightGBM && \
cd LightGBM && mkdir build && cd build && \
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ .. && \
make -j && \
cd ../python-package && python setup.py install --precompile
# NLP
pip install -U nltk
# https://www.nltk.org/data.html
# default location: /usr/share/nltk_data
# use -d flag to specify a different location
python -m nltk.downloader all
pip install Unidecode
pip install jieba
pip install wordcloud
pip install vaderSentiment
#!/bin/bash
sudo apt-get update --fix-missing
# gdebi replace dpkg to solve dependencies during installation
sudo apt install -y gdebi
sudo apt install -y cmake
# communication
# Slack -- ubuntu software
# editor
## vim
sudo apt install -y vim
## zsh
sudo apt-get install zsh
sh -c "$(wget https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh -O -)"
## atom
wget https://atom.io/download/deb -O atom.deb && \
yes | sudo gdebi atom.deb && \
rm atom.deb
# image processing
## image editor
sudo apt install gimp
sudo apt install pinta
## screenshot
sudo apt install flameshot
# gnome editor
sudo apt install dconf-editor
# browser
sudo apt install -y torbrowser-launcher
sudo apt install -y chromium-browser
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -O chrome.deb && \
yes | sudo gdebi chrome.deb && \
rm chrome.deb
# dropbox
wget https://www.dropbox.com/download?dl=packages/ubuntu/dropbox_2015.10.28_amd64.deb -O dropbox.deb && \
yes | sudo gdebi dropbox.deb && \
rm dropbox.deb
# music player
# wget https://github.com/trazyn/ieaseMusic/releases/download/v1.2.9/ieaseMusic-1.2.9-linux-amd64.deb -O ieaseMusic.deb && \
# yes | sudo gdebi ieaseMusic.deb && \
# rm ieaseMusic.deb
wget http://d1.music.126.net/dmusic/netease-cloud-music_1.1.0_amd64_ubuntu.deb -O netease-cloud-music.deb && \
yes | sudo gdebi netease-cloud-music.deb && \
rm netease-cloud-music.deb
# https://github.com/MajeurAndroid/Adb-Remote-Screen/issues/14
# Failed to load module "canberra-gtk-module"
sudo apt install -y libcanberra-gtk-module
# Local file: "" ("netease-cloud-music")
# Suspend from both command line and desktop -> ```sudo netease-cloud-music``` or click shutdown then cancel
# input
sudo apt install -y \
fcitx-bin \
fcitx-googlepinyin
# use " since there is characters &
wget "http://cdn2.ime.sogou.com/dl/index/1524572264/sogoupinyin_2.2.0.0108_amd64.deb?st=qC_O2p5443g1a2TJR_rSdA&e=1533163019&fn=sogoupinyin_2.2.0.0108_amd64.deb" -O sogoupinyin.deb && \
yes | sudo gdebi sogoupinyin.deb && \
rm sogoupinyin.deb
# reboot -> fcitx configure -> add input
# VPN
## ShadowSocks
pip install git+https://github.com/shadowsocks/shadowsocks.git@master
## openvpn
sudo apt install -y \
openvpn \
network-manager-openvpn \
network-manager-openvpn-gnome
# print directory structure in the form of a tree
sudo apt-get install tree
sudo apt-get install -y openssh-server
sudo apt install net-tools # ifconfig
sudo apt-get install -y nmap
sudo apt-get install -y fail2ban
cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local
# latex
sudo apt install -y texlive-full
# 7zip
sudo apt-get install -y p7zip-full p7zip-rar
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.