Last active
January 12, 2024 15:04
-
-
Save joecorall/c99d604d45c61971fb31c7dbf8bddf16 to your computer and use it in GitHub Desktop.
For a list of PIDs you need to import into i2 from i7, get the metadata from i7 via workbench
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -eou pipefail | |
# how many items to fetch at once | |
PARALLEL_EXECUTIONS=10 | |
PIDS=($(cat ./pids.csv)) | |
while [ "${#PIDS[@]}" -gt 0 ]; do | |
for ((i = 0; i < PARALLEL_EXECUTIONS; i++)); do | |
array_length=${#PIDS[@]} | |
if [ $array_length -gt 0 ]; then | |
PID="${PIDS[$((array_length-1))]}" | |
unset "PIDS[$((array_length-1))]" | |
else | |
break | |
fi | |
cp configs/missing.yml configs/missing-$i.yml | |
echo "pids:" >> configs/missing-$i.yml | |
echo " - $PID" >> configs/missing-$i.yml | |
echo "csv_output_path: 'missing-$i.csv'" >> configs/missing-$i.yml | |
echo "log_file_path: 'missing-$i.log'" >> configs/missing-$i.yml | |
python3 get_islandora_7_content.py --config configs/missing-$i.yml & | |
job_ids+=($!) | |
done | |
for job_id in "${job_ids[@]}"; do | |
wait "$job_id" || echo "One job failed, but continuing anyway" | |
done | |
if [ ! -f missing.csv ]; then | |
head -1 missing-0.csv > missing.csv | |
fi | |
for ((i = 0; i < PARALLEL_EXECUTIONS; i++)); do | |
tail -n +2 missing-$i.csv >> missing.csv | |
done | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment