Ubuntu common scripts

! Ubuntu commands and scripts

Random Ubuntu commands and scripts

certbot_letsencrypt.sh

# replace certbot with certbot-auto depending on your installation

# setup
sudo apt install certbot python3-certbot-nginx
sudo certbot --nginx --cert-name example.com -d example.com -d www.example.com

# add
sudo certbot certificates # find Certificate Name eg. example.com
sudo certbot --nginx --cert-name example.com -d example.com -d www.example.com -d test.example.com # include old and new domain names

# renew (setup in cron every 2 months)
# 0 0 1 */2 * certbot renew
sudo certbot renew 

# manual renewal with dns record verification
sudo certbot certonly --server https://acme-v02.api.letsencrypt.org/directory --manual --preferred-challenges dns --cert-name example.com

# delete
sudo certbot delete

chamilo_nginx.conf

server {
	listen 0.0.0.0:80;
	server_name dlearning.kasihkerja.com;
	access_log /var/log/nginx/dlearning.log;        
	index index.php index.html index.htm;
	root /var/www/dlearning;

	# From http://www.chamilo.org.pe/documentation/installation_guide.html

	charset utf-8;

	location @rewrite{
		rewrite ^certificates/$ certificates/index.php last;
		rewrite ^/courses/([^/]+)/$ /main/course_home/course_home.php?cDir=$1 last;
		rewrite ^/courses/([^/]+)/index.php$ /main/course_home/course_home.php?cDir=$1 last;
		rewrite ^/courses/([^/]+)/scorm/(.*([\.js|\.css|\.png|\.jpg|\.jpeg|\.gif]))$ app/courses/$1/scorm/$2 last;
		rewrite ^/courses/([^/]+)/scorm/(.*)$ /main/document/download_scorm.php?doc_url=/$2&cDir=$1 last;
		# Alternatively, you can choose to give direct access to all SCORM files, which is much faster but less secure
		# rewrite "^/courses/([^/]+)/scorm/(.*)$" /app/courses/$1/scorm/$2 break;

		rewrite "^/courses/([^/]+)/document/certificates/(.*)$" /app/courses/$1/document/certificates/$2 last;
		rewrite ^/courses/([^/]+)/document/(.*)$ /main/document/download.php?doc_url=/$2&cDir=$1 last;
		rewrite ^/courses/([^/]+)/upload/([^/]+)/(.*)$ /main/document/download_uploaded_files.php?code=$1&type=$2&file=$3 last;
		rewrite ^/courses/([^/]+)/work/(.*)$ /main/work/download.php?file=work/$2&cDir=$1 last;
		rewrite ^/courses/([^/]+)/upload/course_home_icons/(.*([\.png|\.jpg|\.jpeg|\.gif]))$ app/courses/$1/upload/course_home_icons/$2 last;
		rewrite ^/courses/([^/]+)/(.*)$ /app/courses/$1/$2 last;
		rewrite ^/session/([^/]+)/about/?$ /main/session/about.php?session_id=$1 last;
		rewrite ^/course/([^/]+)/about/?$ /main/course_info/about.php?course_id=$1 last;
		rewrite ^/badge/(\d+) /main/badge/issued.php?issue=$1 last;
		rewrite ^/skill/(\d+)/user/(\d+)$ /main/badge/issued_all.php?skill=$1&user=$2 last;
		rewrite ^/badge/(\d+)/user/(\d+)$ /main/badge/issued_all.php?skill=$1&user=$2 last;
		rewrite ^/main/exercice/(.*)$ /main/exercise/$1 last;
		rewrite ^/main/newscorm/(.*)$ /main/lp/$1 last;
		rewrite ^/service/(\d+)$ /plugin/buycourses/src/service_information.php?service_id=$1 last;
		rewrite "^/main/upload/users/(.*)/(.*)/my_files/(.*)$" /app/upload/users/$1/$2/my_files/$3 last;

		try_files $uri /index.php$is_args$args;
		break;
	}

	location / {
		try_files $uri @rewrite;
	}

	location /main {
		rewrite ^/main/admin/?$ /main/admin/index.php last;
	}

	location ~ \.php$ {
		client_max_body_size 20M;
		try_files $uri @rewrite;

		fastcgi_pass unix:/var/run/php/php7.3-fpm.sock;
		fastcgi_split_path_info ^(.+\.php)(/.*)$;
		include fastcgi_params;
		fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
		fastcgi_param HTTPS off;
	}

	# Serve static files directly
	location ~* \.(png|jpe?g|gif|ico|js|css|mp3|swf|flv|mp4|ogg|woff|woff2)$ {
		rewrite ^/courses/([^/]+)/course-pic85x85.png$ /app/courses/$1/course-pic85x85.png last;
		rewrite ^/courses/([^/]+)/course-pic.png$ /app/courses/$1/course-pic.png last;
		rewrite ^/courses/([^/]+)/scorm/(.*)$ /main/document/download_scorm.php?doc_url=/$2&cDir=$1 last;
		rewrite ^/courses/([^/]+)/document/(.*)$ /main/document/download.php?doc_url=/$2&cDir=$1 last;
		rewrite ^/courses/([^/]+)/work/(.*)$ /main/work/download.php?file=work/$2&cDir=$1 last;
		rewrite ^/courses/([^/]+)/upload/(.*)$ /app/courses/$1/upload/$2 last;
		# For all these media resources not treated by previous rewrites, give direct access (no permission check)
		rewrite ^/courses/(.*)$ /app/courses/$1 break;
		expires 30d;
		access_log off;
		try_files $uri @rewrite;
	}

	location ~ ~\.(ht|git){
		deny all;
	}
	location ^~ /tests/ {
		deny all;
	}

	# if using proxy such as cloudflare
#	if ($http_x_forwarded_proto = "http") {
#		return 301 https://$host$request_uri;
#	}

	listen 443 ssl; # managed by Certbot
	ssl_certificate /etc/letsencrypt/live/kasihkerja.com/fullchain.pem; # managed by Certbot
	ssl_certificate_key /etc/letsencrypt/live/kasihkerja.com/privkey.pem; # managed by Certbot
	include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
	ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

}

git.sh

# revert mode to origin
git diff -p origin | grep -E '^(diff|old mode|new mode)' | sed -e 's/^old/NEW/;s/^new/old/;s/^NEW/new/' | git apply

htcondor_installation.txt

sudo apt update
sudo apt upgrade
sudo apt install build-essential htcondor openssh-server 
# follow the steps below (“For Master” and “For Slave”) during htcondor installation wizard

# if not yet detected, register master and slave domain name IP in
sudo nano /etc/hosts 

# edit CONDOR_ADMIN, UID_ADMIN, CONDOR_HOST, ALLOW_WRITE, ALLOW_NEGOTIATOR
sudo nano /etc/condor/conf.d/00debconf

# make sure ssh key authentication works between machine in cluster
ssh-keygen
ssh-copy-id user@slave1 (or user@master from slave1)

# make sure master/head allow password authentication because condor_cloud requires it
sudo nano /etc/ssh/sshd_config 
sudo service restart ssh

# start condor service on startup/boot
sudo systemctl enable condor.service (if not working try: sudo update-rc.d condor defaults)
sudo reboot now

# make sure master and slave(s) are detected
sudo condor_status

# Installation wizard steps

# For Master (central)
Manage initial Condor configuration automatically? Yes
Perform a "Personal HTCondor installation"? No
Role of this machine in the HTCondor pool? Job submission, Central Manager
File system domain label? (blank)
User directory domain label? (blank)
Address of the central manager? master (this machine name)
Machines with write access to this host? master, slave1 (slave machine name)

# For Slave (executor)
Manage initial Condor configuration automatically? Yes
Perform a "Personal HTCondor installation"? No
Role of this machine in the HTCondor pool? Job submission, Central Manager
File system domain label? (blank)
User directory domain label? (blank)
Address of the central manager? master (this machine name)
Machines with write access to this host? master, slave1 (slave machine name)

# References
https://www.youtube.com/watch?v=mt5dtDrOt2g&t=403s
https://research.cs.wisc.edu/htcondor/tutorials/fermi-2005/admin/admin_handout/

laravel-ubuntu.sh

#!/bin/bash

# Laravel setup

# Ubuntu 20 LTS - Laravel 8.0

sudo apt update
sudo apt install mysql-server nginx
sudo apt install php7.4-common php7.4-bcmath openssl php7.4-json php7.4-mbstring

misc.sh

# remove <none> docker images
docker rmi $(docker images -f "dangling=true" -q)

# generate basic auth
echo -ne "admin:secret" | base64 --wrap=0

# curl POST json
curl -X POST \
  -H "Content-Type: application/json" \
  -H "Authorization: Basic $AUTH" \
  -d  '{"key1":"value1", "key2":"value2"}' \
  https://example.com/

moodle-nginx-cloudflare.conf

server {
        listen 0.0.0.0:80;
        server_name moodle.*;
        access_log /var/log/nginx/moodle.log;
        
        # if using proxy such as cloudflare
	    if ($http_x_forwarded_proto = "http") {
		    return 301 https://$host$request_uri;
	    }

        index index.php index.html index.htm;

        root /var/www/moodle;
        location ~ [^/]\.php(/|$) {
                fastcgi_split_path_info  ^(.+\.php)(/.+)$;
                fastcgi_index index.php;
                fastcgi_pass unix:/var/run/php/php7.1-fpm.sock;
                include fastcgi_params;
                fastcgi_param PATH_INFO $fastcgi_path_info;
                fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
        }

        listen 443 ssl; # managed by Certbot
        ssl_certificate /etc/letsencrypt/live/moodle/fullchain.pem; # managed by Certbot
        ssl_certificate_key /etc/letsencrypt/live/moodle/privkey.pem; # managed by Certbot
        include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
        ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

	    location ~ /\.ht {
		    deny all;
	    }
}

nginx-laravel-config

server {
	listen 80;
	listen [::]:80; 	
	server_name example.com;
	return 301 https://$host$request_uri;
}

server {
	listen 443 ssl http2;
	listen [::]:443 ssl http2; 	
	server_name example.com;

	# if ($scheme = http) {
        #	return 301 https://$host$request_uri;
        # }          

	# ssl
	ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; # managed by Certbot
	ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; # managed by Certbot
	include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
	ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
  
	# improve HTTPS performance with session resumption
	ssl_session_cache shared:SSL:10m;

	root /var/www/example.com;
	index index.php index.html index.htm;

	# compression
	gzip on;
	gzip_vary on;
	gzip_disable "msie6";
	gzip_comp_level 6;
	gzip_min_length 1100;
	gzip_buffers 16 8k;
	gzip_proxied any;
	gzip_types text/plain text/css text/js text/xml text/javascript application/javascript application/x-javascript application/json application/xml application/xml+rss;

	location / {
		try_files $uri $uri/ /index.php$is_args$args;
	}

	# php
	location ~ \.php$ {
		try_files $uri =404;
		fastcgi_split_path_info ^(.+\.php)(/.+)$;
		fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
		fastcgi_index index.php;
		include fastcgi.conf;
		# fix cache issue
                fastcgi_param SCRIPT_FILENAME $realpath_root$fastcgi_script_name;
                fastcgi_param DOCUMENT_ROOT $realpath_root;
	}

	# static dirs
	location /doc {
		alias /var/www/example.com/apidoc;
		index index.html;
		autoindex on;
	}

	# websocket (ratchet)
	location /ws {
		proxy_pass http://localhost:8090;
		proxy_http_version 1.1;
		proxy_set_header Upgrade $http_upgrade;
		proxy_set_header Connection "Upgrade";
	}

	# prevent unimportant logs
	location = /favicon.ico { log_not_found off; access_log off; }
	location = /robots.txt { log_not_found off; access_log off; allow all; }
	location ~* \.(css|gif|ico|jpeg|jpg|js|png)$ {
		expires max;
		log_not_found off;
	}

	location ~ /\.ht {
		deny  all;
	}
}

nginx-mysql-php.sh

# php
PHPVER=7.2 sudo apt-get install php$PHPVER-fpm php$PHPVER-mysql php$PHPVER-cli php$PHPVER-xml php$PHPVER-gd php$PHPVER-mbstring php$PHPVER-zip php$PHPVER-curl php$PHPVER-opcache
PHPVER=7.4 sudo apt-get install php$PHPVER-fpm php$PHPVER-mysql php$PHPVER-cli php$PHPVER-xml php$PHPVER-gd php$PHPVER-mbstring php$PHPVER-zip php$PHPVER-curl

# composer
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
php -r "if (hash_file('sha384', 'composer-setup.php') === '8a6138e2a05a8c28539c9f0fb361159823655d7ad2deecb371b04a83966c61223adc522b0189079e3e9e277cd72b8897') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
sudo php composer-setup.php --install-dir=/bin/ --filename=composer
php -r "unlink('composer-setup.php');"

nginx-vuejs-nodejs.config

server {
	listen 80 ;
	listen [::]:80 ;
	server_name example.com;

  # force redirect http to https
	if ($host = example.com) {
		return 301 https://$host$request_uri;
	} # managed by Certbot

	return 404; # managed by Certbot
}

server {

	listen [::]:443 ssl default_server ipv6only=on; # managed by Certbot
	listen 443 ssl default_server; # managed by Certbot
	server_name example.com; # managed by Certbot
	
	ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; # managed by Certbot
	ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; # managed by Certbot
	include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
	ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

  # vuejs dist
	root /var/www/example;
	index index.html index.htm index.nginx-debian.html;
	location / {
		try_files $uri $uri/ /index.html;
	}

  # nodejs API
	location /api {
		proxy_pass http://localhost:3000;
		proxy_http_version 1.1;
		proxy_set_header Upgrade $http_upgrade;
		proxy_set_header Connection 'upgrade';
		proxy_set_header Host $host;
		proxy_set_header X-Real-IP $remote_addr;
		proxy_set_header X-Forwarded-Proto $scheme;
		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
		proxy_cache_bypass $http_upgrade;
	}
}

python_source_install.sh

#!/bin/bash
# references: 
# * https://tecadmin.net/install-python-3-7-on-ubuntu-linuxmint/
# * https://stackoverflow.com/questions/57743230/userwarning-could-not-import-the-lzma-module-your-installed-python-is-incomple

sudo apt-get install build-essential checkinstall libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev liblzma-dev
cd /usr/src
sudo wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz
sudo tar xzf Python-3.7.4.tgz
cd Python-3.7.4
sudo ./configure --enable-optimizations
sudo make altinstall

ubuntu.sh

# lazy git routine
git pull origin master & git add -A . && git commit -a -m "first commit" && git push origin master

# important files
/etc/rc.local # place for boot up scripts
/home/user/.bashrc # place for environment vars setup
/etc/sudoers # enable sudo without password: username     ALL=(ALL) NOPASSWD:ALL
/etc/ssh/sshd_config # enable/disable password login

# route request from 80 to 8080
iptables -t nat -A PREROUTING -p tcp --dport 80 -j REDIRECT --to 8080
# to remove
iptables -t nat -D PREROUTING -p tcp --dport 80 -j REDIRECT --to 8080

# mitigate single ip dos attack
iptables -A INPUT -p tcp --syn --dport 80 -m connlimit --connlimit-above 50 -j REJECT
# to remove
iptables -D INPUT -p tcp --syn --dport 80 -m connlimit --connlimit-above 50 -j REJECT

# setup git first time
git config --global user.email "yohanes.gultom@gmail.com"
git config --global user.name "Yohanes Gultom"
git config --global credential.helper store


# search file content recursively
grep -r somefilecontent *

# find and replace all recursively with extension filter
find /some/path/to/search -name \*.fileextension -exec sed -i "s/old/new/g" {} \;

# convert mkv to mp4
sudo apt-get install libav-tools
avconv -i input.mkv -codec copy output.mp4

# batch convert mkv to mp4
# sudo apt-get install libav-tools
#!/bin/sh
findpath=$1
: ${findpath:="."}
find "$findpath" -name '*.mkv' | while read f ; do
    dir=$(dirname "$f");
    file=$(basename "$f");
    # ext="${filename##*.}";
    name="${file%.*}";
    # echo "avconv -i \"$f\" -codec copy \"$dir/$name.mp4\"";
    avconv -i "$f" -codec copy "$dir/$name.mp4" </dev/null;
    # rm -f "$f";
done


# cut mp4 video
ffmpeg -i orignal.mp4 -ss 00:00:00 -t 00:00:04 -async 1 -strict -2 cut.mp4

# cut mp3 starting from 00:00:30 to 00:01:40 (100s)
ffmpeg -ss 30 -t 70 -i inputfile.mp3 -acodec copy cut.mp3

# Solve Locale issue on fresh Debian-based installation
locale-gen
export LC_ALL="en_US.UTF-8" # put in .bashrc

# SSH keys
ssh-copy-id user@hostname.example.com

# Merging pdfs
pdftk *.pdf cat output merged.pdf

# check big dirs
du -h . | grep '[0-9\.]\+G'
du -a -h --max-depth=1 | sort -hr

# find deleted files that are not yet released be services
lsof | grep deleted |tee -a outputfile

# backup restore postresql with sudo
sudo -i -u postgres pg_dump dbname > /path/to/backup.sql
sudo -i -u postgres psql dbname < /path/to/backup.sql

# backup/restore mysql
mysqldump -u root -p dbname > "backup_$(date +'%Y%m%d%H%M').sql"
mysqldump -u root -p dbname | gzip -9 > "backup_$(date +'%Y%m%d%H%M').sql.gz"
gzip -d backup.sql.gz
mysql -u root -p dbname < backup.sql
mysql -u root -p dbname -B -e "select * from table;"| sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > table.csv
mysqlimport --ignore-lines=1 --fields-terminated-by=, --columns='id,col1,col2,col3' --local -u root -p dbname table.csv
mysqladmin drop db; mysqladmin create db; mysql db < backup.sql

# find modified file in last 2 days
find . -mtime -2 -ls > findlatest.txt

# delete file older than 5 days
find . -mtime +5 -exec rm {} \;

# run command in the background (still running even after quitting shell)
nohup some_command > some_command.out 2>&1 &
nohup some_command > some_command.out 2>&1 & echo $! > some_command.pid

# find top 10 ram-eating processes
ps aux --sort=-%mem | awk 'NR<=10{print $0}'

# find user's home
grep username /etc/passwd | cut -d ":" -f6

# set dirs and files permissions
find . -type d -exec chmod 755 {} \;
find . -type f -exec chmod 644 {} \;

# clean old logs
sudo find /var/log/ -type f -regex '.*\.[0-9]+\.gz$' -delete

# empty file using sudo
sudo -u www-data bash -c '>storage/logs/laravel.log'

# clear /boot
sudo dpkg --list 'linux-image*'|awk '{ if ($1=="ii") print $2}'|grep -v `uname -r` | while read -r line; do sudo apt-get -y purge $line;done;sudo apt-get -y autoremove; sudo update-grub

# lets encrypt on nginx
# register new certificate
sudo certbot-auto --nginx -d example.com -d www.example.com
# add subdomain to existing certificate
sudo certbot-auto --nginx --cert-name example.com -d example.com -d www.example.com -d test.example.com

# copy heroku config
# https://emirkarsiyakali.com/heroku-copying-environment-variables-from-an-existing-app-to-another-9253929198d9
heroku config -s -a existing-heroku-app > config.txt
cat config.txt | tr '\n' ' ' | xargs heroku config:set -a new-heroku-app

# load env variables from key=value file
# https://stackoverflow.com/questions/19331497/set-environment-variables-from-file-of-key-value-pairs
export $(grep -v '^#' .env | xargs -d '\n')

# scan and fix bad blocks
sudo badblocks -v /dev/sda1 > ~/bad_sectors.txt
sudo fsck -l bad_sectors.txt /dev/sda1 # fat32
sudo e2fsck -l bad_sectors.txt /dev/sda1 # ext2-4

ufw_setup.sh

# standard web server
sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw allow ssh
sudo ufw allow out ssh
sudo ufw allow http
sudo ufw allow out http
sudo ufw allow https
sudo ufw allow out https
sudo ufw allow ntp # NTP to keep the system clock
sudo ufw allow out ntp
sudo ufw allow 53 # dns
sudo ufw allow out 53
sudo ufw enable

xorg.conf

# /etc/X11/xorg.conf for SIS 771/671 as a work around to enable proper resolution since no official driver available
# tested on ubuntu 16.04 LTS

Section "Device"
  Identifier "Generic Video Card"
    VendorName  "Silicon Integrated Systems [SiS]"
        BoardName   "771/671 PCIE VGA Display Adapter"
    Busid "PCI:1:0:0"
    Driver "vesa"
    Screen 0
        Option "UseFBDev" "true"
        Option "DPMS"
        Option "ShadowFB"
        Option "MaxXFBMem"
        VideoRam 262016
        Option "RenderAccel" "true"
        Option "AllowGLXWithComposite" "true"
        Option "backingstore" "true"
        Option "AddARGBGLXVisuals" "True"

EndSection

Section "Monitor"
    Identifier    "Configured Monitor"
    Vendorname    "Generic LCD Display"
    Modelname    "LCD Panel 1280x800"
    HorizSync 20-107
        VertRefresh 50-185
  modeline  "800x600@56" 36.0 800 824 896 1024 600 601 603 625 +hsync +vsync
  modeline  "800x600@60" 40.0 800 840 968 1056 600 601 605 628 +hsync +vsync
  modeline  "1280x768@60" 80.14 1280 1344 1480 1680 768 769 772 795 -hsync +vsync
  modeline  "1280x720@60" 74.48 1280 1336 1472 1664 720 721 724 746 -hsync +vsync
  modeline  "1280x800@60" 83.46 1280 1344 1480 1680 800 801 804 828 -hsync +vsync
    Gamma    1.0
EndSection

Section "Screen"
    Identifier    "Default Screen"
    Monitor        "Configured Monitor"
    Device        "Configured Video Device"
    Defaultdepth    24
    SubSection "Display"
        Depth    24
        Virtual    1280    768
        Modes        "1280x768@60"    "1280x720@60"    "800x600@60"    "1280x800@60"    "800x600@56"
    EndSubSection
EndSection

Section "Module"
    Load "dri"
    Load "dbe" # Double-Buffering Extension
    Load "v4l" # Video for Linux
    Load "extmod"
    Load "type1"
    Load "freetype"
    Load "glx" # 3D layer
    Load "GLcore"
    Load "i2c"
    Load "bitmap"
    Load "ddc"
    Load "int10"
    Load "vbe"
    Load "speedo"
    Load "record"
EndSection

Section "DRI"
        Mode 0666
EndSection