Some useful bash commands

gistfile1.txt

# some bash tricks (Ubuntu 16.04)

## show the smallest 500 files
ll | awk '{print $5" "$NF}' | sort | head -500

## if find some empty files, you can output them to a file named empty_files.txt
## then rm them using this pipe
cat empty_files.txt | awk '{print "index/"$2}' | xargs rm

## rank file size from largest to smallest and show top 100
ll | awk '{print $5" "$NF}' | sort -nr | head -100

## check whether jpeg files are corrupted ('sudo apt install jpeginfo' first)
find -iname “*.jpg” -print0 | xargs -0 jpeginfo -c | grep -e WARNING -e ERROR

## open a long list of files and do remote copying using 'scp'
cat test_list | awk '{print $1" 141:/mnt/triple139/data/google/test/"}' | xargs scp

## split a file without splitting the text lines
#split mydata.txt into 3 sub files and use the prefix 'mydata.part'
split -n l/3 mydata.txt mydata.part 
#combine them 
cat mydata.part* > mydata.txt

## Emptying the buffers cache (run as root)
## note that this does not improve performance (see https://www.linuxatemyram.com/)
sudo sh -c 'echo 3 >/proc/sys/vm/drop_caches'

## rsync with creating new sub-folders.
rsync -a --relative /new/x/y/z/ user@remote:/pre_existing/dir/