prepare_s3.sh

File Edit Options Buffers Tools Sh-Script Help                                                                                                                                                                                   
#!/bin/bash                                                                                                                                                                                                                      
pushd data/sets/ > /dev/null
mkdir -p data/sets/
mkdir -p archives
find . -maxdepth 1 -mindepth 1 -type d -not -name archives -not -name data | while read -r set
do
  export set set=`basename $set`
  touch -d "`find $set -exec stat \{} --printf="%y\n" \; | sort -n -r | head -1`" ${set}test
  if [ ${set}test -nt archives/$set.tar.gz ]
  then
    ln -sd `pwd`/$set data/sets/$set
    tar -chvf archives/$set.tar.gz data/ > /dev/null
    echo "Compressed dataset \"$set\" to data/sets/archives/$set.tar.gz"
    rm data/sets/$set
  else
    echo "$set.tar.gz up to date!"
  fi
  rm ${set}test
done
rm -rf data
popd > /dev/null
echo "Complete!"
tar -czf datasets.tar.gz data/sets
s3cmd -P sync -rr datasets.tar.gz s3://blockplayer/datasets.tar.gz