useful unix commands

commands.txt

# display file with line numbers
cat file -n

# display gzipped file with line numbers
zcat file | cat -n

# display file from the i. line to j. line (that is j minus i there)
cat file | head -n j | tail -n j-i

# remove first line of the file (efficient)
tail -n +2 file > new_file

# remove first line of the file (less efficient)
sed 1d file > newfile

# show from a specific line number <number> till the end of the file
tail -n +<number> file

# count how many lines of a file contain a pattern
cat file | grep pattern | wc -l
grep pattern -c file

# select non-matching lines
cat file | grep -v pattern

# get the number of the first line where a pattern occur
cat file -n | grep pattern -m 1 | cut -f1 | tr -d ' '

# display N lines after, before, around pattern
grep -A N pattern file
grep -B N pattern file
grep -C N pattern file

# display N lines around pattern (only first appearance, useful if you know that pattern is unique)
grep -m 1 -C N pattern file

# adding together two (or more) files and create a new file from the result
cat file1 file2 > newfile

# create new compressed tar archive
tar -zcvf archive.tar.gz dirname

# extract from tar archive
tar -xzf archive.tar.gz

# view an tar archive
tar tvf archive.tar.gz

# create uncompressed tar archive
tar -cvf archive.tar dirname

# extract uncompressed tar archive
tar -xvf archive.tar dirname

# extract tar.bz2
tar jxf filename.tar.bz2

# add user
sudo adduser <username>

# add group
sudo groupadd <groupname>

# add existing user to group
sudo usermod -a -G <groupname> <username>

# add a user to multiple groups
sudo usermod -a -G <groupname1>,<groupname2>,<groupname3> <username>

# view user's group assignemnts
id <username>
groups <username>

# make a directory accessible for a group
# change group of all files/directories recursively
sudo chgrp -R <groupname> <directory>
# add write permissions to group
sudo chmod -R g+w <directory>
# set "GID", so that all new files and directories created under <directory> are owned by the group
sudo find <directory> -type d -exec chmod 2775 {} \;

# randomize lines of a big file memory efficiently
uuid -v 4 -n $(wc -l < lines.txt) | paste - lines.txt | sort | cut -f2 > lines_random.txt 

# change hostname
sudo vim /etc/hostname -> change old name to new name
sudo vim /etc/hosts -> change old name to new name
sudo /etc/init.d/hostname restart

# search a device for bad blocks
sudo badblocks -v <device_name>
sudo badblocks -v /dev/md2

# renice (reset priority of processgroup): niceness: 0 (normal) - 19 (low)
sudo renice -n <niceness> -g <pid>

# count and filter csv files
# remove header (first line), the separator in csv file is ","; count the rows where the value in the second column is above a threshold value (4.5)
tail -n +2 file.csv | awk -F "," '$2 >= <my threshold value>' | wc -l

# pg_dump ssh tunnel
ssh -o "Compression=no" mydbserver "pg_dump -Fc -Z9 -U postgres mydb" > mydb.dump
ssh -i <private_key_path> -o "Compression=no" yourdomain.com "pg_dump --username=<username> --host=localhost --blobs --verbose --format=c --compress=9 <db_name>" > mydb.dump

# generate random password
# of course there are tons of ways to do this
# you can replace 32 with the desired password length
< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;

# force the user to change their password upon next login
sudo chage -d 0 <username>

# server shutdown
sudo shutdown -h now
sudo poweroff

# server restart / reboot
sudo reboot

# encrypt/decrypt file with openssl
openssl aes-256-cbc -salt -in <file> -out <encrypted_file> -k <passphrase>
openssl aes-256-cbc -d -in <encrypted_file> -out <file> -k <passphrase>

# encrypt/decrypt file with openssl NEW
openssl enc -aes-256-cbc -md sha512 -pbkdf2 -iter 1000 -salt -in <file> -out <encrypted_file> -k <passphrase>
openssl aes-256-cbc -d -md sha512 -pbkdf2 -iter 1000 -in <encrypted_file> -out <file> -k <passphrase>

# generate your dhparam.pem file
openssl dhparam -out /etc/nginx/ssl/dhparam.pem 4096

# search for pattern in codebase
grep -inIEr --color=ALWAYS "<pattern_to_search>" .

# search for pattern in files with specific extension (.py in the example)
find . -iname '*.py' | xargs grep -inIE --color=ALWAYS "<pattern_to_search>"

# copy directory tree recursively without overwrite
rsync -a -v --ignore-existing <src_dir> <dst_dir>

# find processes with most memory usage
ps -e -orss,%mem,cputime,%cpu,pid,args | sort -b -k1,1n | pr -TW$COLUMNS

# find processes with maximum file descriptors
lsof -Fpcn | nawk '/^p/ { pid=substr($0,2) } /^c/ { cmd=substr($0,2) } /^n/ { fd[cmd"["pid"]"]++ } END  { for (cc in fd) printf("%-30s %i\n",cc,fd[cc]) } ' | sort -n -k 2 | tail -30

# copy file with rsync over SSH 
rsync -avz -e "ssh -i <ssh_key_path> -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" --progress <file> <username>@<remote_ip>:<remote_dir>

# count files in a directory recursively
find . -type f | wc -l

# adds newline at the end of the file if it doesn't already end with a newline
sed -i -e '$a\' file

# remove CTRL-M (^M) characters from a file
sed -e "s/\r//g" file > newfile

# check for all visible characters
cat -A file

# generate ssh key
ssh-keygen -t rsa -b 4096 -C "<optional_comment>"

# check listening ports and applications
sudo lsof -i -P -n | grep LISTEN

# git | delete all local branches that are gone
# will not delete branches that are not fully merged
git branch -vv | grep gone | cut -d " " -f3 | xargs git branch -d

# check TLS connections
# TLS1.1
openssl s_client -connect example.com:443 -tls1_1
# TLS1.2
openssl s_client -connect example.com:443 -tls1_2
# TLS1.3
openssl s_client -connect example.com:443 -tls1_3

# list all the processes currently listening
ss -ltup

# delete files older than 180 days in the current directory tree
find . -type f -mtime +180 | xargs rm