Linux and terminal useful commands sh bash

0-limits.md

System

View system open file limit

$ cat /proc/sys/fs/file-max

Change default service limits

To increase default 1048576 add following parameter:

$ vi /etc/sysctl.conf
...
fs.nr_open = 2000000

View & edit user limits

vi /etc/security/limits.conf

# Set # of open files limit to 2000000
sudo sed -i 's/*[[:blank:]]\+soft[[:blank:]]\+nofile[[:blank:]]\+[0-9]\+/* soft nofile 2000000/g' /etc/security/limits.conf
sudo sed -i 's/*[[:blank:]]\+hard[[:blank:]]\+nofile[[:blank:]]\+[0-9]\+/* hard nofile 2000000/g' /etc/security/limits.conf

Process

View open file limits of a process

cat /proc/<pid>/limits

Make sure process service has proper open file limit

$ vi /etc/systemd/system/<my service>.service
...
[Service]
LimitNOFILE=2000000

1_1_dns-cache-dnsmasq.md

DNS cache with dnsmasq

Install dnsmasq

sudo yum install dnsmasq

Configure dnsmasq

cat <<EOF | sudo tee /etc/resolv.dnsmasq.conf
nameserver <First Default DNS Server IP>
nameserver <Second Default DNS Server IP>
EOF

Use dnsmasq as local DNS and set configurations

cat <<EOF | sudo tee -a /etc/dnsmasq.conf
listen-address=127.0.0.1

cache-size=2048
resolv-file=/etc/resolv.dnsmasq.conf

###
# Example:
# server=/mydomain.com/1.1.1.1
###
server=/<custom domain>/<custom domain first dns>
server=/<custom domain>/<custom domain second dns>

server=/<second custom domain>/<second custom domain first dns>
server=/<second custom domain>/<second custom domain second dns>
EOF

sudo systemctl enable dnsmasq
sudo systemctl start dnsmasq

sudo rm -f /etc/resolv.conf.save

cat <<EOF | sudo tee /etc/resolv.conf
options timeout:2 attempts:3 single-request-reopen
nameserver 127.0.0.1
EOF


cat <<EOF | sudo tee /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
BOOTPROTO=dhcp
ONBOOT=yes
DNS1=127.0.0.1
DNS2=127.0.0.1
EOF

sudo service network restart
sudo systemctl restart nscd.service

1_2_dns-cache-systemd-resolved.md

DNS cache with systemd-resolved

Edit /etc/systemd/resolved.conf

~# vi /etc/systemd/resolved.conf
[Resolve]
DNS=<first domain dns> <second domain dns>
FallbackDNS=<first default dns> <second default dns>
Domains=<first domain> [<second domain> etc.]
Cache=yes

Restart systemd-resolved

systemctl restart systemd-resolved

Check configuration is loaded

systemd-resolve --status

2_disk-operations.md

List disks with mountpoints

$ lsblk

NAME               MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
nvme0n1            259:1    0   20G  0 disk 
`-nvme0n1p1        259:2    0   20G  0 part /
nvme1n1            259:0    0  200G  0 disk 
`-vg_use           253:0    0  100G  0 lvm  /u01
sda                259:0    0    1T  0 disk
vdb                259:0    0   50G  0 disk

Creating disk partition

$ fdisk /dev/sda

# create new partition
Command (m for help): n

# primary partition
> p

# partition number 1
> 1

# 'enter' to create first sector with default size
> <enter>

# 'enter' to use default disk range
> <enter>

# write changes to disk
Command (m for help): w

Format disk/partition:

mkfs.ext4 ${device path; disk [if not partitioned] or partition, example: /dev/sda or /dev/sda1}

If it was mounted before, unmount & remove old confiiguration in fstab

umount "${disk}"
sed -i "#\b${disk}\b#d" /etc/fstab

Add new configuration to fstab

sudo mkdir -p "$mountPoint"
echo "${disk} "${mountPoint}" ext4 defaults 0 0" >> /etc/fstab

Mount

# manual mount a disk
mount "${disk/partition name or mountPoint}"

# mount all disks defined in `fstab`
mount -a

Resize disk

if disk not partitioned:

# make sure kernel updates new disk information by executing `partprobe`
partprobe

# Resize disk
resize2fs /dev/vdb

if disk partitioned:

# make sure kernel updates new disk information by executing `partprobe`
partprobe

# grow parition & resize partition
growpart /dev/vdb 1
resize2fs /dev/vdb1

ab-load-test.sh

# post_loc.txt contains the json you want to post
# -p means to POST it
# -H adds an Auth header (could be Basic or Token)
# -T sets the Content-Type
# -c is concurrent clients
# -n is the number of requests to run in the test

ab -p body.txt -T application/json -H 'Authorization: Token abcd1234' -c 10 -n 2000 http://example.com/api/v1/locations/

add-nameserver.md

Add Name Server

Ubuntu

1- Add nameserver ( 10.0.0.0 etc) to ethernet interface in /etc/network/interfaces

iface eth0 inet dhcp
    dns-nameservers 10.0.0.0 <other nameservers seperated by a space>

2- Open /etc/resolvconf/resolv.conf.d/tail and remove rotate keyword

options timeout:2 attempts:3 single-request-reopen

3- Restart networking service

systemctl restart networking.service

Example usage:

cat /etc/resolv.conf \
&& read -n 1 -s -r -p "Press any key to continue" \
&& echo "dns-nameservers 10.0.0.0 100.100.2.136 100.100.2.138" >> /etc/network/interfaces \
&& cat /etc/network/interfaces \
&& echo "--------" \
&& cat /etc/resolvconf/resolv.conf.d/tail \
&& read -n 1 -s -r -p "Press any key to continue" \
&& echo "options timeout:2 attempts:3 single-request-reopen" > /etc/resolvconf/resolv.conf.d/tail \
&& systemctl restart networking.service \
&& cat /etc/resolv.conf

CentOS

1- Remove rotate keyword from /etc/resolv.conf->options

options timeout:2 attempts:3 single-request-reopen

2- Add DNS name servers to /etc/sysconfig/network-scripts/ifcfg-eth0

cat <<EOF >> /etc/sysconfig/network-scripts/ifcfg-eth0
heredoc> DNS1=10.0.0.0
DNS2=100.100.2.136
DNS3=100.100.2.138
EOF

3- Restart network and dns cache services

# Please do not switch order of the commands
sudo service network restart
sudo systemctl restart nscd.service

4- Done!

cd.sh

# Save folder that terminal located in, to variable
tmp_dir=$(pwd)
# Go to current working file directory
cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# Restore previous terminal folder
cd "$tmp_dir"

certificate.sh

# view all public certificates
openssl s_client -showcerts -connect <ip or host>:<port> </dev/null

check-inuse-files.sh

# view number of open files
cat /proc/sys/fs/file-nr

check-port-is-reachable.sh

# Needs Ctrl + C
telent <host> <port>

# CLI friendly: (wait for max 3 seconds)
# linux:
nc -v -z -w 3 <host> <port> &> /dev/null && echo "Online" || echo "Offline"

# mac:
nc -v -z -G 3 <host> <port> &> /dev/null && echo "Online" || echo "Offline"

convert-pdf-to-jpg.sh

# Using Ghostscript
# Convert PDF to JPG (JPEG)
gs -dNOPAUSE -sDEVICE=jpeg -r300 -sOutputFile="/<path to output jpq>/doc%03d.jpg" /<path to input>/some.pdf

create-daemon.sh

sudo apt-get install daemon

# Create a daemon from script
daemon --name="appname" --stdout=log.txt sh ~/commands.sh

curl-ftp.sh

# Upload file to ftp with curl
curl -T /<path-to-file> ftp://<remote host>/<path-to-target-file>  --user <username>:<password>

# Delete file from ftp with curl
curl ftp://<remote-host> --user <username>:<password> -Q "-DELE /<path-to-file>" 

# Rename file in ftp with curl
curl ftp://<remote-host> --user <username>:<password> -Q "-RNFR /<previous-file-name>"  -Q "-RNTO /<new-file-name>"

find-file-in-zip-files-in-a-directory.sh

# Find file in zip or jar files in a directory
find . -name "*.jar" -type f | while read FILE; do unzip -l $FILE | grep -i "<file or class name>" && echo "$FILE"; done

find.sh

# List file with `absolute path``
find `pwd` -name <file-name>

get-last-used-command.sh

# Get last command
$ !!

# Get last ssh command
$ !ssh

# Run last command as root
$ sudo !!

install-ps-command.sh

# install ps command
yum install procps
apt install procps

jq.md

Convert to string

jq ". | tostring"

keys-shortcuts.md

1- ctrl + r backward command search, Ctrl + s forward command search

2- ctrl + x + e open current command in the default text editor

3- ctrl + c interrupt execution of currently running command.

4- ctrl + d this shortcut has two different functionalities:

• delete a word starting from current character in forward.

  • hexdump -C key.pub consider that cursor is the begining of the word key. Result will be hexdump -C .pub. Cursor will stay at the beginning of the point character.

• If there isn't any typed command on terminal (when it's empty), it will behave like logout and/or exit command to logout from current terminal session.

5- ctrl + z suspend a command. use bg to continue it.

6- ctrl + l clear screen (basically it's moves current line to top of the terminal)

7- ctrl + a go to beginning of the current line. ctrl + e go to end of the current line.

8- Meta key shortcuts:

• meta + b or meta + <left_arrow> move cursor 1 word backward.

• meta + f or meta + <right_arrow> move cursor 1 word forward.

• meta + u make current word UPPER case.

  • abc will be ABC

• meta + c capitalize current word.

  • abc will be Abc. ABC will be Abc.

• meta + t transpose two words according to a point character

  • ABC.123 will be 123.ABC

• meta + <backspace> remove a word

9- !! run last command

10- sudo !! run last command with sudo

11- !<number> run <number> indexed command in the history

12- !cat run last cat command

13- add :p to end of exclamation command to print it

• !cat:p will print last cat command

nslookup.sh

# find ip addresses of a domain
$ nslookup collect.trendyol.com
Server:		192.168.1.1
Address:	192.168.1.1#53

Non-authoritative answer:
Name:	collect.trendyol.com
Address: 8.209.80.30
Name:	collect.trendyol.com
Address: 47.91.95.144
Name:	collect.trendyol.com
Address: 8.209.81.57

prevent-concurrent-crontab.md

#
# `crontab` ignore if previous execution doesn't finish
#   
#   flock : simple locking mechanism
#
#   -x  exclusive lock (default)
#   -n  non-blocking: fail rather than waiting
#

01 * * * * /bin/flock -xn /home/hadoop/run-all.lock -c /home/hadoop/run-all.sh
01 * * * * /bin/flock -xn /home/hadoop/run-all.lock -c "/home/hadoop/run-all.sh <some-params>"

ps.sh

# view processes
ps aux

# sort by time
ps aux --sort -time

rescan-disk.md

Fix CentOS Disk Space is Wrong After File Deletion

To refresh available disk space, apply following operations:

$ find /sys -name "rescan"
/sys/bus/pci/rescan
/sys/devices/pci0000:00/0000:00:00.0/rescan
/sys/devices/pci0000:00/0000:00:01.0/rescan
/sys/devices/pci0000:00/0000:00:01.1/rescan
/sys/devices/pci0000:00/0000:00:01.2/rescan
/sys/devices/pci0000:00/0000:00:01.3/rescan
/sys/devices/pci0000:00/0000:00:02.0/rescan
/sys/devices/pci0000:00/0000:00:03.0/rescan
/sys/devices/pci0000:00/0000:00:04.0/rescan
/sys/devices/pci0000:00/0000:00:05.0/rescan
/sys/devices/pci0000:00/0000:00:06.0/rescan
/sys/devices/pci0000:00/0000:00:07.0/rescan
/sys/devices/pci0000:00/pci_bus/0000:00/rescan

$ df -h
Filesystem      Size  Used Avail Use% Mounted on
...
/dev/vdb         50G   50G   0G  100% /mnt/disk1
...

# Check PCI path
$ udevadm info -q all -n /dev/vdb | grep DEVPATH
E: DEVPATH=/devices/pci0000:00/0000:00:06.0/virtio3/block/vdb

# And finally we can rescan it:
$ echo 1 > /sys/devices/pci0000:00/0000:00:06.0/rescan

$ df -h
Filesystem      Size  Used Avail Use% Mounted on
...
/dev/vdb         50G   13G   37G  27% /mnt/disk1
...

resolve-hostname.sh

# resolve hostname with specific dns server
dig A <hostname> @<dns_server_ip>

# What is my ip? #whatismyip
dig +short myip.opendns.com @resolver1.opendns.com

run-command-as-a-user.sh

# run command as a user
sudo -u <username> <command: no wrappers!>

# for example
sudo -u root echo "test"

set-default-text-editor.sh

export EDITOR='<program>'

# for example
export EDITOR='vim'

set-timezone.sh

timedatectl set-timezone Europe/Istanbul

sftp.sh

# Login into remote machine
sftp <user>@<host-or-ip>

# Upload file
sftp> put -pr <local-file-or-folder-path> <remote-dest-folder>

# Download file
sftp> get -pr <remote-file-or-folder-path> <local-dest-folder>

# Exit
sftp> bye

ssh.sh

# create .ssh folder
mkdir .ssh
touch .ssh/authorized_keys

# Generate ssh private key, public key
ssh-keygen -t rsa

# Remove a host from `known hosts`
sed -i '/<host>.*/d' /home/<user>/.ssh/known_hosts

# Add a host to known hosts
ssh-keyscan <host> >> /home/<user>/.ssh/known_hosts

# Login remote terminal
ssh <user>@<host-or-ip>


# fix .ssh permissions
chmod 700 ~/.ssh
chmod 600 ~/.ssh/*
chmod 644 -f ~/.ssh/*.pub ~/.ssh/authorized_keys ~/.ssh/known_hosts

# Exit
exit
# or 
logout

systemctl-add-service.md

Create a service file

cat <<EOF >> /etc/systemd/system/<my service>.service
[Unit]
Description=<my service name>
After=syslog.target network.target

[Service]
LimitNOFILE=65535
User=<user to run service>
PIDFile=<pid file location>
ExecStartPre=<make things to be done before running service>
ExecStart=<command to start>
ExecStop=<command to stop>
ExecReload=<command to restart>
Restart=always

[Install]
WantedBy=multi-user.target
EOF

For additional documentation: (https://www.freedesktop.org/software/systemd/man/systemd.service.html)

Set permissions for the service

chmod 644 /etc/systemd/system/<my service>.service

Reload systemctl

systemctl daemon-reload

Enable Service

systemctl enable <my service>.service

Start the Service

systemctl start <my service>.service

tar.sh

# tar.gz
tar -cvzf tarballname.tar.gz itemtocompress

tcpdump.sh

# listen a port on the host machine
tcpdump -n -S -s 0 -A 'tcp dst port <port>'

temporary-block-client.sh

# Temporarry block client ip

# install tcpkill
yum install dsniff -y

# find client IPs connected to a specific port
netstat -tn 2>/dev/null | grep :<port number> | awk '{print $5}' | cut -d: -f1 | sort -nr | uniq

# block temporary send/receives from that client ip
tcpkill host <host>

# block temporary send/receives from specific port
tcpkill port <port>

vi.sh

# enable syntax highlighting
:syntax on

# disable compatible mode
:set nocompatible

view-number-of-opened-files.sh

# view number of files opened by user
$ sudo lsof -Fn -u <username> | sort  | uniq | grep -e "^n/" | wc -l
<number of files opened will be printed here>

vim-tricks

# After a permission error, save the file with super user permissions
:w !sudo tee %