Generate celery task commands for serializing IDR images to avro

cellmorph-calc.sh

#!/bin/bash

set -eu
shopt -s nullglob

AVRO_PARENTDIR=/uod/idr/homes/szleo/features/idr0012-fuchs-cellmorph/screenA/input
IDR_WRITEABLE=/home/idr-scratch/szleo
RUNAS_USER=13500
ADD_TASK="../venv/bin/python tasks.py"
TASKID_LOG=taskid-calc.log
TILE_WIDTH=168
TILE_HEIGHT=128

RUN=0
if [ "${1:-}" = "-run" ]; then
    RUN=1
fi

if [ ! -f tasks.py ]; then
    echo "ERROR: Must be run from the celery-example directory"
    exit 2
fi
if [ -f "$TASKID_LOG" ]; then
  echo "ERROR: \$TASKID_LOG=$TASKID_LOG already exists, delete or rename it"
  exit 2
fi

echoruntee() {
    echo "$@"
    if [ $RUN -eq 1 ]; then
        "$@" | tee -a "$TASKID_LOG"
    fi
}

count=0
TESTING_LIMIT=0
for parent in "${AVRO_PARENTDIR}/"*; do
    for avro in "${parent}"/*.avro; do
	screen_name="$(basename "${parent}")"
	avro_bn="$(basename "${avro}")"
	log_tag="${avro_bn%.*}"
        count=$((count + 1))
        echoruntee ${ADD_TASK} \
           "${IDR_WRITEABLE}/out/${screen_name}/${log_tag}.out" \
	   "${IDR_WRITEABLE}/out/${screen_name}/${log_tag}.err" \
            docker run --rm -u "${RUNAS_USER}" \
            -v /uod/idr:/uod/idr:ro \
            -v "${IDR_WRITEABLE}:/scratch" \
            simleo/pyfeatures \
            calc \
            "${avro}" \
            -l -W ${TILE_WIDTH} -H ${TILE_HEIGHT} \
            -o "/scratch/out/${screen_name}"
        # Use = so that we can easily disable by setting TESTING_LIMIT=0
        if [ "$TESTING_LIMIT" = "$count" ]; then
            break 2
        fi
    done
done

cellmorph-serialize.sh

#!/bin/bash

set -eu

IDR_METADATA=/uod/idr/homes/scratch/szleo/idr-metadata/idr0012-fuchs-cellmorph/screenA
#IDR_TSV=idr0012-screenA-plates.tsv
IDR_TSV=idr0012-screenA-plates.tsv.small
IDR_WRITEABLE=/home/idr-scratch/szleo
#RUNAS_USER=13500
RUNAS_USER=1000
ADD_TASK="../venv/bin/python tasks.py"
TASKID_LOG=taskid-serialize.log

RUN=0
if [ "${1:-}" = "-run" ]; then
    RUN=1
fi

if [ ! -f tasks.py ]; then
    echo "ERROR: Must be run from the celery-example directory"
    exit 2
fi
if [ -f "$TASKID_LOG" ]; then
  echo "ERROR: \$TASKID_LOG=$TASKID_LOG already exists, delete or rename it"
  exit 2
fi

echoruntee() {
    echo "$@"
    if [ $RUN -eq 1 ]; then
        "$@" | tee -a "$TASKID_LOG"
    fi
}

while IFS=$'\t' read -r -a arr; do
    screen_name="${arr[0]}"
    screen_file="${arr[1]}"
    echoruntee ${ADD_TASK} \
        "${IDR_WRITEABLE}/out/${screen_name}.out" \
	"${IDR_WRITEABLE}/out/${screen_name}.err" \
        docker run --rm -u "${RUNAS_USER}" \
        -v /uod/idr:/uod/idr:ro \
        -v "${IDR_WRITEABLE}:/scratch" \
        simleo/pyfeatures \
        serialize \
        "${IDR_METADATA}/${screen_file}" \
        -o "/scratch/out/${screen_name}"
done < "${IDR_METADATA}/${IDR_TSV}"

features-generate-commands.sh

#!/bin/bash

set -eu
shopt -s nullglob

AVRO_PARENTDIR=/uod/idr/scratch/features-input-avro/idr0013-neumann-mitocheck/idr0013-neumann-mitocheck_screenA_ts-6-46-86
IDR_WRITEABLE=/home/idr-scratch/spli
RUNAS_USER=11904
ADD_TASK="../venv/bin/python tasks.py"
TASKID_LOG=taskid-calc.log
CALC_PARAMS="-l -W 672 -H 512 -x 672 -y 512"

RUN=0
if [ "${1:-}" = "-run" ]; then
    RUN=1
fi

if [ ! -f tasks.py ]; then
    echo "ERROR: Must be run from the celery-example directory"
    exit 2
fi
if [ -f "$TASKID_LOG" ]; then
  echo "ERROR: \$TASKID_LOG=$TASKID_LOG already exists, delete or rename it"
  exit 2
fi

echoruntee() {
    echo "$@"
    if [ $RUN -eq 1 ]; then
        "$@" | tee -a "$TASKID_LOG"
    fi
}

count=0
TESTING_LIMIT=0
for parent in "$AVRO_PARENTDIR/"*/; do
    for avro in "$parent"/*.avro; do
        outdir="$(basename $parent)"
        logbasename="$outdir/$(basename ${avro%.avro})"
        count=$((count + 1))
        echoruntee $ADD_TASK \
            docker run --rm -u "$RUNAS_USER" \
            -v /uod/idr:/uod/idr:ro \
            -v "$IDR_WRITEABLE:/scratch" \
            manics/pyfeatures \
            --stdout "/scratch/out/$logbasename-out.log" \
            --stderr "/scratch/out/$logbasename-err.log" \
            calc \
            "$avro" \
            $CALC_PARAMS \
            -o "/scratch/out/$outdir"
        # Use = so that we can easily disable by setting TESTING_LIMIT=0
        if [ "$TESTING_LIMIT" = "$count" ]; then
            break 2
        fi
    done
done

mitocheck-generate-commands.sh

#!/bin/bash

set -eu

IDR_METADATA=/uod/idr/homes/scratch/spli/idr-metadata
IDR_TSV=idr0013-neumann-mitocheck/screenA/idr0013-screenA-plates.tsv
IDR_WRITEABLE=/home/idr-scratch/spli
RUNAS_USER=11904
ADD_TASK="../venv/bin/python tasks.py"
TASKID_LOG=taskid-serialize.log

RUN=0
if [ "${1:-}" = "-run" ]; then
    RUN=1
fi

if [ ! -f tasks.py ]; then
    echo "ERROR: Must be run from the celery-example directory"
    exit 2
fi
if [ -f "$TASKID_LOG" ]; then
  echo "ERROR: \$TASKID_LOG=$TASKID_LOG already exists, delete or rename it"
  exit 2
fi

echoruntee() {
    echo "$@"
    if [ $RUN -eq 1 ]; then
        "$@" | tee -a "$TASKID_LOG"
    fi
}

while IFS=$'\t' read -r -a arr; do
    screen_name="${arr[0]}"
    screen_file="${arr[1]}"
    echoruntee $ADD_TASK \
        docker run --rm -u "$RUNAS_USER" \
        -v /uod/idr:/uod/idr:ro \
        -v "$IDR_WRITEABLE:/scratch" \
        manics/pyfeatures \
        --stdout "/scratch/out/$screen_name-out.log" \
        --stderr "/scratch/out/$screen_name-err.log" \
        serialize \
        "$IDR_METADATA/$screen_file" \
        -o "/scratch/out/$screen_name" \
        -ts 6,46,86
done < "$IDR_METADATA/$IDR_TSV"