cleanup_forecasts.sh

#!/bin/bash

set -x
HOME=/data/geoglows
WORKFLOW_DIR=$HOME/output/forecasts

# create date variables
TODAY=$(date +"%Y%m%d")
DATE_LIMIT=$(date -d "$TODAY - 46 days" +"%Y%m%d")
# delete old forecasts
while read watershed
do
# check that there are outputs in the watershed folders
if [[ `find $WORKFLOW_DIR/$watershed/ -mindepth 1 -type d | wc -l` > 0 ]]
then
while read rawdate
do
# delete forecasts older than date limit
if [[ "$watershed" != *"historical"* && "${rawdate:0:8}" < "$DATE_LIMIT" ]]
then
rm -r $WORKFLOW_DIR/$watershed/$rawdate
fi
done < <(ls -d $WORKFLOW_DIR/$watershed/*/ | xargs -n 1 basename)
fi
done < <(ls -d $WORKFLOW_DIR/*/ | xargs -n 1 basename)

# create date variables
DATE_LIMIT=$(date -d "30 days ago" +"%Y%m%d")
# Delete old .zarr files in $WORKFLOW_DIR/v2
find "$WORKFLOW_DIR/v2/" -name "*.zarr" -type f | while read file; do
    file_date=$(basename "$file" | sed 's/[^0-9]*//g')
    # only the first 8 characters are the date
    file_date=${file_date:0:8}
    if [ "$file_date" -lt "$DATE_LIMIT" ]; then
        rm -r "$file"
    fi
done