Last active
February 23, 2021 12:21
-
-
Save F5Training/f6e75eb60d21fb2155d6b9f85898aede to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# | |
# Install required apt packages | |
# | |
REQUIRED_PKG=(apt-transport-https \ | |
zsh \ | |
ansible \ | |
powerline \ | |
python3 \ | |
python3-powerline \ | |
python3-paramiko \ | |
python3-venv \ | |
python3-boto3 \ | |
python3-passlib \ | |
python3-docopt \ | |
python3-ruamel.yaml \ | |
python3-flake8 \ | |
flake8 \ | |
python-passlib \ | |
expect \ | |
jq \ | |
httpie \ | |
python3-docopt \ | |
python3-ruamel.yaml \ | |
python3-jinja2 \ | |
python3-magic \ | |
python-is-python2 ) | |
echo "${USER} ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/99-${USER} > /dev/null 2>&1 | |
sudo touch /etc/apt/apt.conf.d/proxy.conf | |
echo "Acquire::http::Proxy \"http://192.168.1.47:3142/\";" | sudo tee /etc/apt/apt.conf.d/proxy.conf > /dev/null 2>&1 | |
sudo locale-gen en_SG.UTF-8 | |
sudo update-locale LANG=en_SG.UTF8 | |
sudo apt update; sudo apt install -y --no-install-recommends ${REQUIRED_PKG[@]} | |
# | |
# Make required directories | |
# | |
mkdir ~/.config | |
mkdir ~/Projects | |
mkdir -p ~/.local/bin | |
mkdir -p ~/.local/man | |
touch ~/.z | |
# | |
# Download msysgit2unix-socket.py | |
# | |
curl -SL https://gist.githubusercontent.com/F5Training/42f013e63d9d46e69b9c1d4253d64f3e/raw/7b2d6884262b97380f95b66abb8ac76123fa749d/msysgit2unix-socket.py -o ~/.local/bin/msysgit2unix-socket.py | |
chmod +x ~/.local/bin/msysgit2unix-socket.py | |
echo "Type the following:" | |
echo "export SSH_AUTH_SOCK=\"/tmp/.ssh-auth-sock\"" | |
echo "export MSYS_SOCK_PATH=" | |
echo "~/.local/bin/msysgit2unix-socket.py \$MSYS_SOCK_PATH:\$SSH_AUTH_SOCK" | |
# | |
# Create runme1st.sh | |
# | |
cat <<EOF > ~/runme1st.sh | |
#!/bin/bash | |
cd ~/.config | |
if [[ -z \$MYREPO ]]; then | |
echo "MYREPO not set. Please set MYREPO and re-run this script." | |
exit 1 | |
fi | |
git clone git@\${MYREPO} --recurse-submodules | |
cd ~/.config/dotdrop | |
./dotdrop.sh install --profile demo | |
# fzf | |
git clone --depth 1 https://github.com/junegunn/fzf.git ~/.fzf | |
alias curl='curl --silent' | |
yes | ~/.fzf/install | |
EOF | |
# | |
# Create runme3rd.sh | |
# | |
cat <<EOF > ~/runme3rd.sh | |
#!/bin/bash | |
if [[ -z \$LABAUTO ]]; then | |
echo "LABAUTO not set. Please set LABAUTO and re-run this script." | |
exit 1 | |
fi | |
REQUIRED_PKG=(make \\ | |
build-essential \\ | |
libssl-dev zlib1g-dev \\ | |
libbz2-dev \\ | |
libreadline-dev \\ | |
libsqlite3-dev \\ | |
wget \\ | |
curl \\ | |
llvm \\ | |
libncurses5-dev \\ | |
xz-utils \\ | |
tk-dev \\ | |
libxml2-dev \\ | |
libxmlsec1-dev \\ | |
libffi-dev \\ | |
liblzma-dev) | |
sudo apt update; sudo apt install -y --no-install-recommends \${REQUIRED_PKG[@]} | |
if [ \$? != 0 ]; then | |
echo "apt update failed. Re-run this script '\$0' again." | |
exit 127 | |
fi | |
git clone --depth 1 https://github.com/pyenv/pyenv.git ~/.pyenv | |
PYENV_MODULES=(doctor \\ | |
installer \\ | |
update \\ | |
virtualenv \\ | |
which-ext) | |
for module in "\${PYENV_MODULES[@]}"; | |
do | |
git clone --depth 1 https://github.com/pyenv/pyenv-\${module}.git ~/.pyenv/plugins/pyenv-\${module} | |
done | |
if [[ "\$PATH" != *"\$HOME/.local/bin"* ]]; then | |
export PATH="\$HOME/.local/bin:\$PATH" | |
fi | |
export PYENV_ROOT="\$HOME/.pyenv" | |
export PATH="\$PYENV_ROOT/bin:\$PATH" | |
PYTHON_VER=\$(pyenv install -l | grep "\s3\." | egrep -v "dev|a[0-9]|b|rc" | sort -r | head -1) | |
pyenv install \${PYTHON_VER} | |
pyenv global \${PYTHON_VER} | |
eval "\$(pyenv init -)" | |
pip install --upgrade pip | |
PIP_INSTALLS=(wheel \\ | |
docopt \\ | |
ruamel.yaml \\ | |
jinja2 \\ | |
python-magic \\ | |
"--user pipx") | |
for pip in "\${PIP_INSTALLS[@]}"; | |
do | |
pip install \${pip} | |
done | |
PIPX_INSTALLS=(pipenv \\ | |
http-prompt \\ | |
httpie \\ | |
ipython \\ | |
black \\ | |
pytest \\ | |
pytube) | |
for pipx in "\${PIPX_INSTALLS[@]}"; | |
do | |
pipx install \${pipx} | |
done | |
# mkdir -p Projects | |
cd ~/Projects | |
git clone git@\${LABAUTO} | |
cd ~/Projects/lab-automation | |
git checkout v14.1-dev | |
git checkout v14.1 | |
# pipenv install | |
# echo layout pipenv >> .envrc | |
EOF | |
# | |
# Create 1-setup.py | |
# | |
cat <<EOF > ~/1-setup.py | |
#!/usr/bin/env python3 | |
# -*- coding: utf-8 -*- | |
""" | |
Spyder Editor | |
This is a temporary script file. | |
""" | |
from pathlib import Path, PurePosixPath | |
from requests import get | |
from ruamel.yaml import YAML | |
import re | |
from urllib.parse import urlsplit | |
import hashlib | |
import os | |
import pprint | |
def load_data(): | |
""" | |
""" | |
yaml_str = """ | |
cli: | |
github: exercism | |
name: cli | |
regex_str: 'exercism-.*-linux-x86_64\.tar\.gz' | |
content_type: application/gzip | |
checksum: true | |
checksum_file: exercism_checksums.txt | |
checksum_sig: exercism_checksums.txt.sig | |
checksum_type: 'multiline|normal' | |
starship: | |
github: starship | |
name: starship | |
regex_str: 'starship-x86_64-unknown-linux-gnu\.tar\.gz' | |
content_type: application/gzip | |
checksum: true | |
checksum_ext: sha256 | |
checksum_type: 'singleline|checksum_only' | |
yq: | |
github: mikefarah | |
name: yq | |
regex_str: 'yq_linux_amd64' | |
content_type: application/octet-stream | |
checksum: true | |
checksum_file: checksums | |
checksum_order: 18 | |
checksum_type: 'multiline|ordered' | |
task: | |
github: go-task | |
name: task | |
regex_str: 'task_linux_amd64\.tar\.gz' | |
checksum: true | |
checksum_file: task_checksums.txt | |
checksum_type: 'multiline|normal' | |
direnv: | |
github: direnv | |
name: direnv | |
regex_str: 'direnv\.linux-amd64' | |
content_type: application/octet-stream | |
fd: | |
github: sharkdp | |
name: fd | |
regex_str: 'fd-.*-x86_64-unknown-linux-gnu\.tar\.gz' | |
content_type: application/gzip | |
bat: | |
github: sharkdp | |
name: bat | |
regex_str: 'bat-.*-x86_64-unknown-linux-gnu\.tar\.gz' | |
content_type: application/gzip | |
k9s: | |
github: derailed | |
name: k9s | |
regex_str: 'k9s_Linux_x86_64\.tar\.gz' | |
content_type: application/gzip | |
checksum: true | |
checksum_file: checksums.txt | |
checksum_type: 'multiline|normal' | |
kubectx: | |
github: ahmetb | |
name: kubectx | |
regex_str: 'kubectx_.*_linux_x86_64\.tar\.gz' | |
content_type: application/gzip | |
checksum: true | |
checksum_file: checksums.txt | |
checksum_type: 'multiline|normal' | |
kubens: | |
github: ahmetb | |
name: kubectx | |
regex_str: 'kubens_.*_linux_x86_64\.tar\.gz' | |
content_type: application/gzip | |
checksum: true | |
checksum_file: checksums.txt | |
checksum_type: 'multiline|normal' | |
""" | |
return YAML(typ='safe').load(yaml_str) | |
def json_extract(obj, key): | |
"""Recursively fetch values from nested JSON.""" | |
arr = [] | |
def extract(obj, arr, key): | |
"""Recursively search for values of key in JSON tree.""" | |
if isinstance(obj, dict): | |
for k, v in obj.items(): | |
if isinstance(v, (dict, list)): | |
extract(v, arr, key) | |
elif k == key: | |
arr.append(v) | |
elif isinstance(obj, list): | |
for item in obj: | |
extract(item, arr, key) | |
return arr | |
values = extract(obj, arr, key) | |
return values | |
def get_tag_name(data): | |
return data.get('tag_name', '*') | |
def get_repos_mapping(data): | |
repos = {} | |
for key, value in data.items(): | |
repos[key] = dict(github=value['github'], name=value['name']) | |
return repos | |
def get_json(data): | |
return data.json() | |
def retrieve_github_download_urls(data): | |
gh_url = "https://api.github.com/repos/{}/{}/releases/latest" | |
headers = {"Accept":"application/vnd.github.v3+json"} | |
repos = get_repos_mapping(data) | |
for software, value in repos.items(): | |
github = value.get('github') | |
name = value.get('name') | |
resp = get(gh_url.format(github, name), headers=headers) | |
resp_json = get_json(resp) | |
tag_name = get_tag_name(resp_json) | |
browser_download_urls = json_extract(resp_json, 'browser_download_url') | |
data.get(name)['tag_name'] = tag_name | |
regex = data.get(software).get('regex_str') | |
urls = [] | |
for url in browser_download_urls: | |
filename = PurePosixPath(urlsplit(url)[2]).name | |
matched = re.findall(regex, filename) | |
if matched: | |
urls.append(url) | |
data[software]['urls'] = urls | |
return data | |
def mkdir(dir): | |
dir = Path(dir) | |
if not dir.exists(): | |
dir.mkdir() | |
return dir | |
def download_file(url, dir): | |
local_filename = dir / Path(url.split('/')[-1]) | |
# NOTE the stream=True parameter below | |
with get(url, stream=True) as r: | |
r.raise_for_status() | |
with open(local_filename, 'wb') as f: | |
for chunk in r.iter_content(chunk_size=8192): | |
# If you have chunk encoded response uncomment if | |
# and set chunk_size parameter to None. | |
# if chunk: | |
f.write(chunk) | |
return local_filename | |
def download_files(dir, data): | |
for name, item in data.items(): | |
subdir = dir / Path(name) | |
subdir = mkdir(subdir) | |
for file in item.get('urls'): | |
download_file(file, subdir) | |
if item.get('checksum'): | |
if filename := item.get('checksum_file'): | |
url = item.get('urls')[0] | |
url = url[:url.rfind('/')] | |
download_file(f'{url}/{filename}', subdir) | |
print(f'{name}-{item.get("tag_name")}') | |
def verify_checksum(item): | |
status = 'passed' | |
urls = item.get('urls') | |
file_name = urls[0].split('/')[-1] | |
checksum_type = item.get('checksum_type') | |
with open(file_name, 'rb') as f: | |
file = f.read() | |
file_hash = hashlib.sha256() | |
file_hash.update(file) | |
if checksum_type == 'multiline|normal': | |
with open(item.get('checksum_file'), 'r') as f: | |
lines = f.readlines() | |
for line in lines: | |
if file_name in line: | |
if file_hash.hexdigest() != line.split(' ')[0]: | |
print(f'{item.get("name")} checksum failed') | |
status = 'failed' | |
elif checksum_type == 'singleline|checksum_only': | |
with open(file_name + '.sha256', 'r') as f: | |
checksum = f.read().rstrip() | |
if file_hash.hexdigest() != checksum: | |
print(f'{item.get("name")} checksum failed') | |
status = 'failed' | |
elif checksum_type == 'multiline|ordered': | |
with open(item.get('checksum_file', 'r')) as f: | |
lines = f.readlines() | |
for line in lines: | |
if file_name in line: | |
if file_hash.hexdigest() != line.split(' ')[item.get('checksum_order')]: | |
print(f'{item.get("name")} checksum failed') | |
status = 'failed' | |
return status | |
def verify_checksums(data, dir_name): | |
pwd = os.getcwd() | |
status = {} | |
for name, item in data.items(): | |
if item.get('checksum') == True: | |
subdir_name = dir_name / name | |
os.chdir(subdir_name) | |
result = verify_checksum(item) | |
os.chdir(pwd) | |
status[name] = result | |
else: | |
status[name] = 'no checksum' | |
return status | |
if __name__ == '__main__': | |
data = load_data() | |
data = retrieve_github_download_urls(data) | |
dir_name = mkdir('tmp') | |
download_files(dir_name, data) | |
results = verify_checksums(data, dir_name) | |
pp = pprint.PrettyPrinter(indent=4) | |
pp.pprint(results) | |
EOF | |
# | |
# Create move-bin.sh | |
# | |
cat <<EOF > ~/move-bin.sh | |
#!/bin/bash | |
PKGS=("cli" \ | |
"starship" \ | |
"yq" \ | |
"direnv" \ | |
"fd" \ | |
"bat" \ | |
"task" \ | |
"k9s" \ | |
"kubectx" \ | |
"kubens" \ | |
) | |
for pkg in "\${PKGS[@]}"; | |
do | |
case \${pkg} in | |
"starship") | |
cd ~/tmp/starship | |
STARSHIP_TGZ=starship-x86_64-unknown-linux-gnu.tar.gz | |
tar -C ~/.local/bin -zxvf \${STARSHIP_TGZ} | |
;; | |
"cli") | |
cd ~/tmp/cli | |
EXERCISM_TGZ=\$(ls exercism-*-linux-*.tar.gz) | |
tar -C ~/.local/bin -zxvf \${EXERCISM_TGZ} exercism | |
;; | |
"yq") | |
cd ~/tmp/yq | |
YQ_BIN=yq_linux_amd64 | |
mv \${YQ_BIN} ~/.local/bin/yq | |
;; | |
"direnv") | |
cd ~/tmp/direnv | |
DIRENV_BIN=direnv.linux-amd64 | |
mv \${DIRENV_BIN} ~/.local/bin/direnv | |
;; | |
"fd") | |
cd ~/tmp/fd | |
FD_TGZ=\$(ls fd-*-x86_64-unknown-linux-gnu.tar.gz) | |
tar -zxvf \${FD_TGZ} --no-anchored --strip-components=1 fd fd.1 | |
mv fd ~/.local/bin/fd | |
mv fd.1 ~/.local/man/fd.1 | |
;; | |
"bat") | |
cd ~/tmp/bat | |
BAT_TGZ=\$(ls bat-*-x86_64-unknown-linux-gnu.tar.gz) | |
tar -zxvf \${BAT_TGZ} --no-anchored --strip-components=1 bat bat.1 | |
mv bat ~/.local/bin/bat | |
mv bat.1 ~/.local/man/bat.1 | |
;; | |
"task") | |
cd ~/tmp/task | |
TASK_TGZ=task_linux_amd64.tar.gz | |
tar -C ~/.local/bin -zxvf \${TASK_TGZ} task | |
;; | |
"k9s") | |
cd ~/tmp/k9s | |
K9S_TGZ=k9s_Linux_x86_64.tar.gz | |
tar -C ~/.local/bin -zxvf \${K9S_TGZ} --no-anchored --strip-components=1 k9s | |
;; | |
"kubectx") | |
cd ~/tmp/kubectx | |
KUBECTX_TGZ=\$(ls kubectx_*_linux_x86_64.tar.gz) | |
tar -C ~/.local/bin -zxvf \${KUBECTX_TGZ} --no-anchored --strip-components=1 kubectx | |
;; | |
"kubens") | |
cd ~/tmp/kubens | |
KUBENS_TGZ=\$(ls kubens_*_linux_x86_64.tar.gz) | |
tar -C ~/.local/bin -zxvf \${KUBENS_TGZ} --no-anchored --strip-components=1 kubens | |
esac | |
# kubectl | |
done | |
KUBECTL_VER=\$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) | |
curl -sSLO "https://storage.googleapis.com/kubernetes-release/release/\${KUBECTL_VER}/bin/linux/amd64/kubectl" | |
mv kubectl ~/.local/bin/kubectl | |
echo kubectl-\${KUBECTL_VER} | |
chmod +x ~/.local/bin/* | |
EOF | |
# | |
# | |
# | |
chmod +x ~/*.{sh,py} |
Added mkdir .config .local/bin
Added python-ispython2 package
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
First commit