bash snippets - these are notes for myself; some found and some I've made

bash-snippets.md

git

get the last commit hash in abbreviated form

 git log -n1 --pretty=format:%h

list files in another branch

git ls-files --with-tree=another-branch

grep through all revisions of a repo

git grep -B1 -A2 AWSLambdaBasicExecutionRole $(git rev-list --all) -- *.tf

write file from rev to stdout

git show 0d4c1dfdd11225bce53a85e9b025f2293d4c20c6:terraform/iam.tf

print file from another branch

git show another_branch:some-file

checkout a file from another branch

git checkout other-branch -- some-file

create an 'all' remote containing all remote urls

git remote add all origin-url:git/repo0.git
git remote set-url --add all repo-url1:git/repo1.git
git remote set-url --add all repo-url2:git/repo2.git

diff a file in different branches

git diff master..my-feature-branch -- some-file

search history for a file

git log --all --full-history -- **/thefile.*
git log --all --full-history -- <path-to-file>

details for file in history

git show <SHA> -- <path-to-file>

get file from another branch

git checkout trunk -- src/some-file.py

restore file from tree previous to

git checkout <SHA>^ -- <path-to-file>

checkout all python file from a previous commit

git ls-tree --name-only -r abcdf1234 | grep '.*\.py$' | \
	while read f; do git checkout abcd1234 -- "$f"; done

create repo with remote gist seeded with README.md

git init
git remote add gist "$(gh gist create README.md)"
git fetch gist
git checkout -f master
git add . && git commit -m "add the rest of the files"
git push

test if git repo has a remote named origin using wildcard match

[[ $'\n'$(git remote)$'\n' == *$'\n'origin$'\n'* ]] && echo match
# ... or with regex
[[ "$(git remote)" =~ (^|[^[:alnum:]_])origin($|[^[:alnum:]_]) ]] && echo match

list files in commit (recursive)

git diff-tree --no-commit-id --name-only -r <sha1> [glob expression]

ignoring files that have been commited

git update-index --skip-worktree <file>

add a remote url to an existing remote

git remote set-url origin --push --add another:repo.git

revert to last commit

git reset HEAD --hard

print local repo-name/current-branch-name as 2 words if in a git repository

git status --porcelain --untracked-files=no &>/dev/null && \
	echo -n "$(basename $(git rev-parse --show-toplevel)) " && \
	git rev-parse --abbrev-ref HEAD

list all files that've been deleted

git log --name-status --no-merges | sed -n 's/^[D]'$'\t''//p' | sort | uniq

list all files that haven't been deleted

git log --name-status --no-merges | sed -n 's/^[^D]'$'\t''//p' | sort | uniq

set up a personal upstream git repository

# ssh-host: a host defined in ~/.ssh/config (see: ssh_config(5))
# camaro: the name of the repo

# on a remote host or local filesystem:
git init --bare camaro.git  # (option: use --shared if there are other contributors)

# on local host cd to camaro directory:
git remote add origin ssh-host:[path/]camaro.git
# (for origin on local filesystem: git remote add origin local/path/to/camaro.git)

git push --set-upstream origin --all
git push --set-upstream origin --tags

xml

xmlstarlet xslt example

xmlstarlet sel --template --match "/some/xpath[]" --value-of ./some_element --nl file.xml

testing xslt with xmlstarlet

xml sel -t -v "translate(/info, ' ', '')" <(echo '<info> ABC  = xyz </info>')

validating sitemap.xml against sitemaps.org schema

xmllint --noout --schema https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd sitemap.xml 2>&1

compare xml files

diff -w <(xmllint --c14n a.xml) <(xmllint --c14n b.xml)

openssl

base 64 encoding and decoding

echo 'hello there' | openssl enc -base64 | openssl enc -base64 -d

create CSR

sudo openssl genrsa -out example.com.key 2048
sudo openssl req -new -key example.com.key.key -out example.com.key.csr

create a self signed certificate

openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/nginx/ssl/nginx.key \
	-out /etc/nginx/ssl/nginx.crt

add an SSHA encrypted entry for user Jim to .htpasswd file

PASSWORD='SEcRe7PwD'; SALT=$(openssl rand -base64 3);
SHA1=$((echo -n "$PASSWORD$SALT" | openssl dgst -binary -sha1; echo -n $SALT) | openssl base64)
echo "Jim:{SSHA}$SHA1" >> .htpasswd

generate an upper case RFC 4122 UUID using openssl and sed

openssl rand -hex 16 | \
	sed 'y/abcdef/ABCDEF/; s/\(........\)\(....\)\(....\)\(....\)\(.*\)/\1-\2-\3-\4-\5/'

generate an RFC 4122 UUID using openssl and bash string operations

U=$(openssl rand -hex 16)
echo ${U:0:8}-${U:8:4}-${U:12:4}-${U:16:4}-${U:20:12}

create a binary sha1 signature of some stuff in a file named "data" (or stdin):

openssl dgst -sha1 -sign mykey.pem -out data.sig.sha1.bin data

verify an sha1 signature with rsa public key and original data file (or stdin):

openssl dgst -sha1 -verify pubkey.pem -signature data.sig.sha1.bin data

miscellaneous

number base conversion (note: hex numbers should be uppercase)

echo "obase=10; ibase=16; 0F" | bc
15

convert yaml to json

python -c 'import yaml; import json; import sys; print(json.dumps(yaml.safe_load(open(sys.argv[1]))))' hello.yaml

variable expansion to change case (bash >= 4)

my_var='AbCDe'
echo ${my_var,,}  # tolower
abcde

echo ${my_var^^}  # toupper
ABCDE

echo ${my_var~~}  # toggle
aBcdE

test if file is binary

# sample the top of the file with `dd`
binCharCount=$( dd count=4 2>/dev/null < "$myFile" | tr -d '[:print:][:space:]' | wc -c )
test "$binCharCount" -gt 0 && echo binary || echo text

test if file does not end with a \n

test -n $(tail -c1 somefile) && echo 'somefile does not end with a \n'

conditional based on presence of a process in a one-liner

ps -C docker >/dev/null && echo "Hey docker!" || echo "Where's docker?"

bash function to get yes/no feed back from the user (return is considered no here)

agree() {
    local question="$1"

    while true; do
        read -rp "$question " answer

        case $answer in
            [Yy]|[Yy][Ee][Ss]) return 0 ;;
            [Nn]|[Nn][Oo]|'') return 1 ;; # empty string is default
            *) echo "Please answer yes or no.";;
        esac
    done
}
agree 'Should we continue?' || exit

use process substitution and a custom file descriptor to read lines from a process

exec 3< <(find . -type f -maxdepth 1)
while read -ru 3 line; do
    : # ...
done

convert alpha phone numbers to decimal

tr "&'(ABCDEFGHIJKLMNOPQSTUVWXYZ" '1112223334445556667778889999'

find results as null terminated strings

find . -type f -print0 | while read -rd $'\0' f; do echo "$f"; done

wildcard word boundary match (note: requires a single word boundry character)

b=' '; [[ "${b}hello there${b}" == *${b}the${b}* ]] && echo match

encode/decode from js console

let str = "Hello People";
let enc = btoa(str);
console.log(enc); // Outputs: "SGVsbG8gUGVvcGxl"
let dec = atob(enc);
console.log(dec); // Outputs: "Hello People"

find files last modified more than a day ago

find /var/log -type d -maxdepth 1 -mtime +1d -print0 | \
	while read -rd $'\0' f; do echo "$f"; done

test if existing file has non space character, eg. a file with printable characters

[[ "$(</tmp/README.md)" == *[^[:space:]]* ]] && echo not empty

sed dry-run

diff file <(sed 's/bird/frog/' file)

regex for matching ip numbers

/( 25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]? ) \. ( 25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]? ) \.
 ( 25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]? ) \. ( 25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]? ) /

sort ip numbers

sort -n -t\. -k1 -k2 -k3 -k4

sort the output of find (GNU) by seconds from epoch, then strip off the leading seconds search key

find . -type f -printf "%T@\t%Tc\t%h/%f\n" | sort -nr | sed 's/^[^\t]*\t//'

start linux VM in headless mode with VBoxManage

VBoxManage list vms
"rhel" {0ab4f939-6785-486c-a174-b67836cdf1a7}
VBoxManage startvm rhel --type headless
# to shutdown log out of any ssh sessions then:
VBoxManage controlvm rhel poweroff

check locally for internet access

netstat -nrf inet | grep -q ^default && \
    echo internet is up || \
    echo internet is down

search LDAP for login name with wildcards

ldapsearch -x -LLL uid='cTiern*' uid | sed -n 's/^uid:[[:blank:]]//p' | uniq

list 10 most recently modified files in the current directory while excluding .git

find . ! -regex '.*/\.git\(/.*\)?$' -printf '%T@\t%TY-%Tm-%Td %Tr\t%h/%f\n' | \
	sort -nr | sed -n '1,10{s/[0-9\.]\{1,\}[[:blank:]]*//p}'

replace runs of 2 or more spaces or horizontal tabs with tabs

sed -e "s/[[:blank:]]\{2,\}/\t/g"

print pid of java process

ps -C java -o pid=

when nginx configtest fails, show why

sudo service nginx configtest || tail -n5 /var/log/nginx/error.log

multi-line batch editing html with perl

perl -i -0777pe 's/my old string(\s*)/my new string$1/s' $(
	find . -name \*.html -exec grep -l 'my old string' "{}" \; 2>/dev/null
)

start popfile spam filter proxy

cd /Library/POPFile/; sudo VERSIONER_PERL_VERSION=5.12 /usr/bin/perl -Ilib popfile.pl

determine if an ios binary contains localizations other than english

unzip -l some.ipa | \
sed '/\/en\.lproj\//d' | \
grep '/..\.lproj/Localizable\.strings' >/dev/null \
&& echo multiple localizations \
|| echo english localization only

dump a mysqlite data base

echo .dump | sqlite3 database.sqlite | less

create a tarball of files on a remote server while saving it locally

ssh remote-host 'tar -cjf - remote-folder' > local-file.tbz

show process that is serving web pages

sudo lsof -i :80

extract bundleidentifier from ios ipa file

unzip -p some.ipa \*/Info.plist | /usr/libexec/PlistBuddy -c 'Print :CFBundleIdentifier' /dev/stdin

list host names of all virtual hosts in apache httpd.conf

awk '/^<VirtualHost/{while ($0 !~ /ServerName/) {getline} print $2}' httpd.conf

test if remote file exists via ssh e.g. before using scp

rfile() { ssh ${1%%:*} "stat ${1##*:} &>/dev/null"; }
# EXAMPLE:
# rfile ssh-host:afile && echo file exists || scp afile ssh-host:afile

open a link in default browser if Firefox not present.

open -a Firefox 'http://localhost' 2>/dev/null || open 'http://localhost'

extract rdf nodes from pdf and output as xml

env LANG=en_US.US-ASCII tr -cs '[:print:]' '\n' < some.pdf | \
awk 'BEGIN {
    print "<?xml version=\"1.0\"?>"
    print "<rdf:RDF"
    print "  xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\""
    print "  xmlns=\"http://purl.org/rss/1.0/\""
    print "  xmlns:admin=\"http://webns.net/mvcb/\""
    print ">"
}
/<rdf:Description/,/<\/rdf:Description/{
    print
}
END {
    print "</rdf:RDF>"
}'

convert pdf to html using doc2html.pl

doc2html.pl /absolute/path/to/some.pdf 'application/pdf'

Get current flood stage for a river (in feet) in the USA. In this example the Winooski river at Essex Junction, VT (essv1).

# TIP: When this value reaches or exceeds 12.5, avoid North Willston Rd. Bridge.
#      It will most likely be closed. ;) --CBT
curl -s 'http://water.weather.gov/ahps2/hydrograph_to_xml.php?gage=essv1&output=xml' | \
xpath '/*/observed/datum[1]/primary/text()' 2>/dev/null; echo \'

test if a variable is a number

isNumber() { [[ "$1" =~ ^[0-9]+$ ]] && echo Is a number || echo Is NOT a number; }

sign a mac application

codesign -s 'Certificate Name' -i 'bundle.identifier' -fv 'Your Program.app'
# windows equivalent:
# signtool sign /v /f c:\path\to\key.p12 /p password /t http://timestamp.example.com prog-to-sign.exe

read a password in perl without echoing characters

perl -e 'use Term::ReadKey; print "password: "; ReadMode 2; $pw=ReadLine; ReadMode 0; print "\n$pw"'

extract the Adobe director player from a mounted shockwave installer disk image, switch to an empty folder then run the following:

gunzip -c /Volumes/Adobe*Shockwave*/Shockwave_Installer_Full.pkg/Contents/Archive.pax.gz | \
pax -r ./Lib*/App*

list read-only mounted file systems

mount | sed -n 's/^\/dev\/.* on \(\/.*\) (.*read-only.*/\1/p'

remove comments

sed 's/[[:blank:]]*#.*//; /^$/d' file

netcat as a port scanner

nc -zw2 localhost 80 &>/dev/null && echo open || echo closed

docker

copy a file from an image without running it

docker create --name temp <image_name>
docker cp temp:/path/to/file /dest/to/file
docker rm -f temp