Shell Snippets

awk_examples.sh

awk -F':' '{ print $1 }' <file>
# Print the first column for a file whose columns are separated by ':'.

awk '$2 == 5'
awk '$2 == 5 { print $0}'
# Print the entire lines where the second column equals 5.
# Other comparison operators: == != < > <= >= ?:

awk '/tom|jerry|vivek/'
# Print Lines Containing tom, jerry AND vivek

commands_to_practice.sh

apropos
Wildcard ?
ctrl+z stops job instead of ctrl+c that kills it
du -hs *  # Disk usage statistics
egrep  # For regular expressions
grep -i  # Ignore case
grep -rnw '/path/to/somewhere/' -e 'pattern'

# Everyday Use
ctrl-u to delete the content from current cursor back to the start of the line
ctrl-k to delete to the end of the line
ctrl-l to clear the screen

compress_all_files_in_folder_to_gzip.sh

echo 'Note that the csv files will be removed'
echo '(Press any key to proceed. Press ctrl + c to abort.)'
read
find . -name '*.csv' -exec gzip {} \;

convert_video_files_quickly.sh

# Example: Convert all the mkv videos in a folder into mp4
cd dir_with_video_files
mkdir converted
for file in *.mkv; do ffmpeg -i $file -codec copy converted/"${file%.*}".mp4; done;

current_script_path.sh

SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"

display_files_with_long_rows_of_text.sh

head -n 300 <file> | less -S

display_first_and_second_column_of_csv_file.sh

cut -d',' -f1 -f2 <file>.csv
# Option `-d` sets the delimeter, in this case a comma, ','.

download_website.sh

# https://stackoverflow.com/a/6510193
wget -nd -r -l1 -P /save/location -A jpeg,jpg http://www.example.com/products

extract_subtitles_from_video_file.sh

# Determine the number of the subtitle track to exctract
mkvinfo <filename>
# Assuming that we chose subtitle track number 3, use the following command to extract the subtitles
mkdir subtitles; for file in *.mkv; do mkvextract tracks $file 3:"subtitles/${file%.*}.srt"; done;

# More information here: https://askubuntu.com/questions/452268/extract-subtitle-from-mkv-files

filter_and_copy_files.py

# Copy png files in <folder> to <new_folder>
find <folder> -name "*.png" -print0 | xargs -0 -I file cp file <new_folder>/file

force_ssh_with_password.sh

ssh -o PreferredAuthentications=password -o PubkeyAuthentication=no user@domain

format_markdown_table_as_plain_text.sh

# This command opens a nano text editor where to insert the Markdown text. Then, it exports the text into plain text (rST, actually) and copies it to the MacOS clipboard.

touch tmp_md_to_txt_export.md; nano tmp_md_to_txt_export.md; pandoc --from markdown --to plain --wrap=none tmp_md_to_txt_export.md | sed '/^$/d' | sed 's/^[ \t]*//' | pbcopy; rm tmp_md_to_txt_export.md

# `sed '/^$/d'` removes empty lines.
# `sed 's/^[ \t]*//'` remove leading whitespaces.

google_cloud_cheatsheet.sh

# gcloud
gcloud projects list
gcloud init
gcloud components update
gcloud components install COMPONENT_ID
gcloud components remove COMPONENT_ID
gcloud components list
gcloud app deploy --project PROJECT_ID -v VERSION_NUMBER
gcloud projects describe <PROJECT_ID>
gcloud app browse
https://<PROJECT_ID>.appspot.com
gcloud config list
gcloud config set project <DEFAULT_PROJECT_ID>
gcloud config configurations list
gcloud config configurations activate <CONFIGURATION_NAME>

# gsutil
gsutil ls
gsutil cp gs://my-bucket/*.txt .

highligh_line_matching_word.sh

# This display every line in the output and only highlights those matching the word.
... | grep --colour=always -E ".*word.*|$"

install_latexit.sh

brew install ghostscript
brew install pdf2svg
brew install --cask basictex
brew install --cask latexit

keyboard_interrupt.sh

#!/bin/bash

# trap ctrl-c and call ctrl_c()
trap ctrl_c INT

function ctrl_c() {
        printf "\n** Keyboard interrupt\n"
        exit 1
}

for i in `seq 1 5`; do
    sleep 5
    echo -n "."
done

list_files_with_size_higher_than_1mb.sh

find . -type f -size +1M -exec ls -lh {} \;

memory_usage.sh

#!/bin/bash -e

echo "      date     time $(free -m | grep total | sed -E 's/^    (.*)/\1/g')"
while true; do
    echo "$(date '+%Y-%m-%d %H:%M:%S') $(free -m | grep Mem: | sed 's/Mem://g')"
    sleep 60
done

nano_keybindings.txt

M-S    Save changes
M-D    Count the number of words, lines, and characters
M-}    Indent the current line
M-{    Unindent the current line
M-3    Comment/uncomment the current line or marked lines

M-L    Hard wrapping of overlong lines enable/disable
M-#    Line numbering enable/disable

^L     Refresh (redraw) the current screen
^Z     Suspend the editor (if suspension is enabled)

M-R    Enable Regex while in text search

pipenv_help.sh

# Fix the command prompt PS1 errors
ln -s ~/.bash_profile ~/.bashrc

# When using 'pipenv shell --python <version>', Pyenv should be available for downloading the version
brew install pyenv

pretty_print_csv.sh

head -n 300 data.csv | sed 's/,/ ,/g' | column -t -s, | less -S
# `sed 's/,/ ,/g'` prevents from merging empty cells into one cell
# If the separator is ";", specify it as `column -t -s;`

pyenv_cheatsheet.sh

# List Python versions available to be installed
pyenv install --list

# Install Python version
pyenv install 3.6.2

# Display Pyenv environments folder
pyenv root

# Display Pyenv environment folder for a certain Python version
pyenv prefix 3.6.2

# Installing a virtual environment with Python 3.6.2
pyenv install 3.6.2
mkvirtualenv --python=$(pyenv prefix 3.6.2)/bin/python <virtualenv_name>
# Alternative
mkvirtualenv --python=$(pyenv root)/versions/3.6.2/bin/python <virtualenv_name>

random_oracle.sh

# The random oracle shuffles a list
python3 -c "import random; l = ['A', 'B', 'C', 'D', 'E']; random.shuffle(l); print(l)"

redirection_of_stdout_and_stderr.sh

> file  # Redirects stdout to file
1> file  # Redirects stdout to file
2> file  # Redirects stderr to file
&> file  # Redirects stdout and stderr to file

remove_broken_symlinks_with_confirmation.sh

find -L . -type l -exec rm -i {} +

reusing_command_output.sh

OUTPUT=$(ls)  # Save output
echo "$(ls)"  # Reuse output

# NOTE: If we did not use the inverted commas, ", and just wrote
#       echo $(ls), all the lines of the output would have been
#       concatenated into one, with the outputs separated by
#       spaces

screen_cheatsheet.sh

# Create a new session:
screen

# Create names session:
screen -S session_name

# Detach screen:
Ctrl+a, d

# Resume last screen session:
screen -r

# List screens:
screen -ls

# Resume a certain screen session
# (session ID = initial numerical ID when listing screens):
screen -r 10835

# Scroll using copy mode:
Ctrl+a, Escape
# Note: Press enter to start selecting text and press enter again to end the selection. 

# Paste the copied text:
Ctrl+a, ]

# Paste the copied text to file:
Ctrl+a, :
hardcopy -h <filename>  # In command mode
# Note: If no text is selected, it copies the whole scroll buffer.

# Close a screen (method 1):
# Enter the screen and type:
exit

# Close a screen (method 2):
screen -Xs 20411 quit

# Close all detached screens:
screen -ls | grep detached | cut -d. -f1 | awk '{print $1}' | xargs -I screen_num screen -Xs screen_num quit

# Show help prompt:
Ctrl+a, ?

# .screenrc file
zombie kr  # Prevent screen from exiting after command finished (press k to kill the screen and r to restart)
verbose on
defscrollback 10000  # Increase the number of scrollback lines

sed_cheatsheet.sh

# Sed tutorial by example

# Example: Substitute "cat" with "dog" in file.txt
sed 's/cat/dog/g' file.txt
# Here "s" stands for "substitute" and "g" for global. If "g" was not present, only the first occurrence would be substituted.
# The pattern after the first slash, /, is the pattern to look for and the pattern after the second slash is the pattern to substitute.

# Example: Substitute parentheses, (), in string 'Hello, (awesome) World!' with underscores, _.
echo 'Hello, (awesome) World!' | sed -E 's/(.*)\((.*)\)(.*)/\1_\2_\3/'
# Out:  Hello, _awesome_ World!
# The -E option allows to use extended regular expressions (ERE). Note that these ERE restrected commands, that are listed in the sed manual.
# The parentheses in the regular expression `(.*)\((.*)\)(.*)` capture the group of characters, which can be used in the substitution part with \1, \2 and \3, where the number indicates the order the the groups.

# Example: Substitute spaces with commas
echo '1 2 3 4' | sed -E 's/ /,/g'
# Out:  1,2,3,4

# Example: Substitue swear words with the profanity symbol, #$@&%*!
SWEAR_WORDS='fudge|motherfather|custard'
echo 'fudge you, custard! You, motherfather!' | sed -E "s/($SWEAR_WORDS)/#\$\@\&\%\*\!/g"
# Out:  #$@&%*! you, #$@&%*!! You, #$@&%*!!
# Be careful with profanity; use the inverted commas! The inverted commas, "...", were used, so that the SWEAR_WORDS variable could be replaced in the string.

shell_function_with_arguments.sh

#!/bin/bash

greeting () {
  echo "Hello $1"
}

greeting "Joe"
# Hello Joe

stack_images.sh

# This needs ImageMagic
brew install imagemagick

# Stack images horizontally
#
#  +-------+-------+-------+
#  |       |       |       |
#  |       |       |       |
#  |       |       |       |
#  |       |       |       |
#  +-------+-------+-------+
#
convert +append 1.tiff 2.tiff 3.tiff 4.tiff out.tiff

# Stack images vertically
#
#  +-------+
#  |       |
#  |       |
#  |       |
#  |       |
#  +-------+
#  |       |
#  |       |
#  |       |
#  |       |
#  +-------+
#  |       |
#  |       |
#  |       |
#  |       |
#  +-------+
#
convert -append 1.tiff 2.tiff 3.tiff 4.tiff out.tiff

stderr_to_stdout.sh

# The stderr is added to the stdout. This is useful for Python loggings.
command 2>&1

# Piping stderr and stdout
command 2>&1 | tee -a log.txt
command |& tee -a log.txt  # Only for Bash 4+

substitute_comma_with_semicolon_in_unquoted_text.sh

sed -Ee :1 -e 's/^(([^",]|"[^"]*")*),/\1;/;t1' file.csv
# Is a very common sed idiom. It keeps doing the same substitution in a loop as long as it's successful.
# Here, we're substituting the leading part of the line made of 0 or more quoted strings or characters other that " and , (captured in \1) followed by a , with that \1 capture and a |, so on your sample that means:
# See: https://unix.stackexchange.com/a/450813

translate_characters.sh

# Tranlsate characters
echo 'Hello, World!' | tr 'a-zA-Z' 'n-za-mN-ZA-M'
# Out: Uryyb, Jbeyq!

# Translate back
echo 'Uryyb, Jbeyq!' | tr 'n-za-mN-ZA-M' 'a-zA-Z'
# Out: Hello, World!

xargs_with_multiple_commands.sh

find . -name <filename> -print0 | xargs -0 -I file sh -c "echo file && echo hello"