Compact multiple PNDA avro files into one

pnda-compact.sh

set -e

for d in $(hdfs dfs -ls -R '/user/pnda/PNDA_datasets/datasets/source=*'|awk '/^d.*hour=..$/ {printf "%s ", $NF}'); do
    IN=$(hdfs dfs -ls ${d}/*.avro | awk '{printf "%s ", $NF}')
    OUT=${d}/compacted.avro.new
    echo "--- In '$d' Compacting files '${IN}' to '${OUT}'"
    hadoop jar /opt/cloudera/parcels/CDH/lib/avro/avro-tools.jar concat ${IN} ${OUT}
    hdfs dfs -chown gobblin:pnda ${OUT}
    hdfs dfs -rm -skipTrash ${d}/*.avro
    hdfs dfs -mv ${OUT} ${d}/compacted.avro
done