This list was posted over at Unix-Ninja. I take no credit for these. Posting these here for my own convenience.
Extract md5 hashes
# egrep -oE '(^|[^a-fA-F0-9])[a-fA-F0-9]{32}([^a-fA-F0-9]|$)' *.txt | egrep -o '[a-fA-F0-9]{32}' > md5-hashes.txt
# egrep -oE '(^|[^a-fA-F0-9])[a-fA-F0-9]{32}([^a-fA-F0-9]|$)' *.txt | egrep -o '[a-fA-F0-9]{32}' > md5-hashes.txt
Note: The above regexes can be used for SHA1, SHA256 and other unsalted hashes represented in hex. The only thing you have to do is change the '{32}' to the corresponding length for your desired hash-type
Extract valid MySQL-Old hashes
# grep -e "[0-7][0-9a-f]\{7\}[0-7][0-9a-f]\{7\}" *.txt > mysql-old-hashes.txt
Extract blowfish hashes
# grep -e "\$2a\\$\08\\$\(.\)\{75\}" *.txt > blowfish-hashes.txt
Extract Joomla hashes
# egrep -o "([0-9a-zA-Z]{32}):(\w{16,32})" *.txt > joomla.txt
Extract VBulletin hashes
# egrep -o "([0-9a-zA-Z]{32}):(\S{3,32})" *.txt > vbulletin.txt
Extraxt phpBB3-MD5
# egrep -o '\$H\$\S{31}' *.txt > phpBB3-md5.txt
Extract Wordpress-MD5
# egrep -o '\$P\$\S{31}' *.txt > wordpress-md5.txt
Extract Drupal 7
# egrep -o '\$S\$\S{52}' *.txt > drupal-7.txt
Extract old Unix-md5
# egrep -o '\$1\$\w{8}\S{22}' *.txt > md5-unix-old.txt
Extract md5-apr1
# egrep -o '\$apr1\$\w{8}\S{22}' *.txt > md5-apr1.txt
Extract sha512crypt, SHA512(Unix)
# egrep -o '\$6\$\w{8}\S{86}' *.txt > sha512crypt.txt
Extract Floating point numbers
# grep -E -o "^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$" *.txt > floats.txt
Extract Social Security Number (SSN)
# grep -E -o "[0-9]{3}[ -]?[0-9]{2}[ -]?[0-9]{4}" *.txt > ssn.txt
Extract Indiana Driver License Number
# grep -E -o "[0-9]{4}[ -]?[0-9]{2}[ -]?[0-9]{4}" *.txt > indiana-dln.txt
Extract US Passport Cards
# grep -E -o "C0[0-9]{7}" *.txt > us-pass-card.txt
Extract US Passport Number
# grep -E -o "[23][0-9]{8}" *.txt > us-pass-num.txt
Extract US Phone Numberss
# grep -Po '\d{3}[\s\-_]?\d{3}[\s\-_]?\d{4}' *.txt > us-phones.txt
Remove the space character with sed
# sed -i 's/ //g' file.txt OR # egrep -v "^[[:space:]]*$" file.txt
Remove the last space character with sed
# sed -i s/.$// file.txt
Sorting Wordlists by Length
# awk '{print length, $0}' rockyou.txt | sort -n | cut -d " " -f2- > rockyou_length-list.txt
Remove blank lines with sed
# sed -i '/^$/d' List.txt
Remove defined character with sed
# sed -i "s/'//" file.txt
Delete a string with sed
# echo 'This is a foo test' | sed -e 's/\<foo\>//g'
Replace characters with tr
# tr '@' '#' < emails.txt OR # sed 's/@/#' file.txt
Print specific columns with awk
# awk -F "," '{print $3}' infile.csv > outfile.csv OR # cut -d "," -f 3 infile.csv > outfile.csv
Note: if you want to isolate all columns after column 3 use # cut -d "," -f 3- infile.csv > outfile.csv
Generate Random Passwords with urandom
# tr -dc 'a-zA-Z0-9._!@#$%^&*()' < /dev/urandom | fold -w 8 | head -n 500000 > wordlist.txt
# tr -dc 'a-zA-Z0-9-_!@#$%^&*()_+{}|:<>?=' < /dev/urandom | fold -w 12 | head -n 4
# base64 /dev/urandom | tr -d '[^:alnum:]' | cut -c1-10 | head -2
# tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 10 | head -n 4
# tr -dc 'a-zA-Z0-9-_!@#$%^&*()_+{}|:<>?=' < /dev/urandom | fold -w 12 | head -n 4 | grep -i '[!@#$%^&*()_+{}|:<>?=]'
# tr -dc '[:print:]' < /dev/urandom | fold -w 10| head -n 10
# tr -cd '[:alnum:]' < /dev/urandom | fold -w30 | head -n2
Remove Parenthesis with tr
# tr -d '()' < in_file > out_file
Generate wordlists from your file-names
# ls -A | sed 's/regexp/&\n/g'
Process text files when cat is unable to handle strange characters
# sed 's/\([[:alnum:]]*\)[[:space:]]*(.)\(\..*\)/\1\2/' *.txt
Generate length based wordlists with awk
# awk 'length == 10' file.txt > 10-length.txt
Merge two different txt files
# paste -d' ' file1.txt file2.txt > new-file.txt
Faster sorting
# export alias sort='sort --parallel=<number_of_cpu_cores> -S <amount_of_memory>G ' && export LC_ALL='C' && cat file.txt | sort -u > new-file.txt
Mac to unix
# tr '\015' '\012' < in_file > out_file
Dos to Unix
# dos2unix file.txt
Unix to Dos
# unix2dos file.txt
Remove from one file what is in another file
# grep -F -v -f file1.txt -w file2.txt > file3.txt
Isolate specific line numbers with sed
# sed -n '1,100p' test.file > file.out
Create Wordlists from PDF files
# pdftotext file.pdf file.txt
Find the line number of a string inside a file
# awk '{ print NR, $0 }' file.txt | grep "string-to-grep"
Again, this list was posted over at Unix-Ninja. I take no credit for these.