Command Line Fun

commands.sh

# XML - get a value using xmlint with xpath the @something (in this case "@string" is an attribute selector
xmllint --xpath "string(//Project/stuff/stuff/Stuff/@string)" some-data.pretty.xml | less

#Select by node index
xmllint --xpath "string(//Project/stuff/stuff[2]/Stuff/@string)" some.pretty.xml | less

# get password for MacOs keycahin assuming that the item is called foobar and the 'account' values is the same as the logged in user
MYPASSWORD=`security find-generic-password -a ${USER} -s foobar -w`

#Pretty print xml
xmllint --format ~/foobar.xml > foobar.pretty.xml

# get a secure note from MacOS keychain
`security find-generic-password -C note -s 'foobar' -w | xxd -r -p | 
 xmllint --xpath "//dict/data/text()" - | base64 --decode | 
 textutil -stdin -convert txt -output ~/Desktop/foobar.txt`

docker.sh

# start a shell session in running container
docker exec -it <container-name> /bin/bash
# see standard out from runnign container
docker logs -f <container-name>
# run a container
docker run -t <container-name> <image-name>
# create an image and name it
docker build -t <image-name> <docker-file>

image_data_to_binary.sh

while getopts fio: opts
do 
    case $opts in
	    f) use_input_file=1
	    input_file=$OPTARGS;;
	    i) use_piped_data=1;;
	    o) output_file=$OPTARGDS;;
    esac
done

if [ use_piped_data -gt 0 ]
    then 
        read -r line
        image_data=$line
        echo "$image_data"
        echo "$output_file"
fi

jq_awk.sh

# Generate a csv from flickr set json
jq -r '.photoset.photo[] | [.title + ".jpg"]  | @csv' data/china-2018-set.json > data/china-uploaded.csv

# Filter based on property value
foojson='{"foo": [{"firstName": "Bob","lastName": "White"},{"firstName": "Dave","lastName": "White"},{"firstName": "Bob","lastName": "Grey"}]}'
echo $foojson  | jq '.foo[] | select(.firstName == "Bob")'

#Sum nested amounts 
salesjson='{"sales":[{"categories": [{"category": "online", "amount": 20.00, "currency": "CNY"}, {"category": "in-store", "amount": 42.00, "currency": "CNY"}]}, {"categories": [{"category": "online", "amount": 14.95, "currency": "USD"}, {"category": "in-store", "amount": 12.00, "currency": "CNY"}]}]
}'

echo $salesjson | jq '[.sales[] | .categories[] | select(.category == "online").amount] | add'

# select by second category criteria also
echo $salesjson | jq '[.sales[] | .categories[] | select(.category == "online") | select(.currency == "CNY").amount] | add'

#Slice up a JSON file to reduce the amount of data for testing
ecommorders='{"sales": [{"categories": [{"category": "online", "amount": 20.00, "currency": "CNY"}, {"category": "in-store", "amount": 42.00, "currency": "CNY"}]}, {"categories": [{"category": "online", "amount": 14.95, "currency": "USD"}, {"category": "in-store", "amount": 12.00, "currency": "CNY"}]}]
, "customers": [{"firstName": "Bob","lastName": "White"},{"firstName": "Dave","lastName": "White"},{"firstName": "Bob","lastName": "Grey"}]}'

echo $ecommorders | jq '{"company": "ACME", "customers":[.customers[]], "sales":[.sales[0:1]]}' > test.json

# sum a csv by column
awk -F ',' '{sum += $6} END {print sum}' store_traffic_metrics_20190124_000000.csv

# Format output of a sum using grep for fuzzy filtering
cat  foo.csv | grep ,01 | awk -F ',' '{sum += $9} END {printf "%.2f\n", sum}'

# Filter and sum using awk
awk -F ',' '{ if ($13 == "01") {sum += $9} } END {printf "%.2f\n", sum}' foo.csv

# Filter with multiple conditions one being date
awk -F ',' '{ if (($13 == "10") && (substr($5,1,10) == "2019-04-22")) {sum += $9} } END {printf "%.2f\n", sum}' foo.csv

python_stuff.sh

# Export a Jupyter Notebook as a python script EXCLUDING any cell that contains a:  # exclude_from_export comment line.
jupyter nbconvert --to script --RegexRemovePreprocessor.patterns="['^# exclude_from_export']" expirement_a.ipynb