arelra

cat keylist.txt | xargs -n 1 -P 4 -I{} sh -c './lister keys/{} > {}.txt;'

arelra

 find . -maxdepth 1 -type f -name 'x*.txt' -printf '%f\n' | xargs cat | parallel -j16 -N 1000 -I{} cp {} --target-directory=/data &

arelra

awk '{print length}' cut.txt | sort -g | uniq -c

arelra

attach
------
aws ec2 attach-volume --volume-id vol-123 --instance-id i-123 --device /dev/sdf --region us-east-1
lsblk (renamed on linux as xv*)
sudo file -s /dev/xvdf (if reports only 'data' then no filesystem)
sudo mkfs -t ext4 /dev/xvdf (ONLY IF NO FILESYSTEM ie NEW VOLUME)
sudo mkdir /data (create mount-point e.g. data)
sudo mount /dev/xvdf /data (mount device at /data)
sudo chmod xxx /data (to add permissions) 
optional: mount volume on instance restart

arelra

sudo apt-get install python-pip
sudo pip install awscli
aws configure

arelra

ls | parallel -j16 -I{} tar -czf /data/keys/{}.tar.gz {} &

arelra

find . -type f -mmin +60 | xargs -I {} aws s3 sync {} s3://....

arelra

diff file1 file2 | awk '{if ( $2 != "") print $2}'

arelra

#!/bin/bash
while read -r line
do
  printf "attempting: $line > "
  tmpfile=./tmp/$line.xml
  aws s3 cp s3://*****/$line $tmpfile > /dev/null
  if grep -q "<???????:para>" $tmpfile; then
    printf "ABSTRACT: Y > "
    xmlstarlet sel -t -o "{ @@@@id@@@@ : @@@@" -v "//?????:eid" -o "@@@@, @@@@abstract@@@@ : @@@@" -t -v "//abstract/?????" -o "@@@@}" $tmpfile > $tmpfile.json
    sed -i -e 's/"/\\"/g' $tmpfile.json

arelra

aws s3 ls s3:// | awk '{ count +=1; size += $3 } END { print "Count:", count, "Size:", size, "bytes" }'