Last active
March 18, 2020 18:06
-
-
Save jaytaylor/16b6f5a7fa2e2e88b46c to your computer and use it in GitHub Desktop.
Busy-box compatible automatic RAR extraction system.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
## | |
# | |
# @author Jay Taylor [@jtaylor] | |
# | |
# @date 2014-12-20 | |
# | |
# @description Busy-box compatible automatic RAR extraction system. | |
# | |
set -e | |
## | |
# Begin Configuration | |
# | |
minAgeInDays=3 | |
maxAgeInDays=180 | |
ageScript='/share/CACHEDEV1_DATA/homes/most-recently-modified.py' | |
# NB: 3 days worth of seconds = 259200. | |
minAgeInSecondsBeforeExtract=$((${minAgeInDays}*60*60*24)) | |
# | |
# End Configuration | |
## | |
function usage() { | |
echo "usage: $0 [directory-name] [?directory-name]..." 1>&2 | |
exit 1 | |
} | |
function abortIfNonZero() { | |
# @param $1 command return code/exit status (e.g. $?, '0', '1'). | |
# @param $2 error message if exit status was non-zero. | |
local rc=$1 | |
local what=$2 | |
test $rc -ne 0 && echo "error: ${what} exited with non-zero status ${rc}" 1>&2 && exit $rc || : | |
} | |
function overrideUtFileNames() { | |
utFiles="$(find . -name '*.\!ut')" | |
if [[ $(echo "${utFiles}" | tr ' ' $'\n' | grep -v '^$' | wc -l) -lt 10 ]]; then | |
for f in ${utFiles}; do | |
mv "${f}" "$(echo "${f}" | sed 's/\.\!ut$//')" | |
done | |
fi | |
} | |
function restoreUtFileNames() { | |
if [[ $(echo "${utFiles}" | tr ' ' $'\n' | grep -v '^$' | wc -l) -lt 10 ]]; then | |
for f in ${utFiles}; do | |
mv "$(echo "${f}" | sed 's/\.\!ut$//')" "${f}" | |
done | |
fi | |
} | |
function collectEligibleDirectories() { | |
directories=() | |
echo -e '\nVerifying specified directories:' | |
for d in $*; do | |
if [[ -z "$(echo "${d}" | grep '^\/' )" ]]; then | |
d="${basePath}/${d}" | |
fi | |
#if [[ -z "$(find "${d}" -type d -maxdepth 0 -mtime +${minAgeInDays} -mtime -${maxAgeInDays})" ]] ; then | |
# echo "skipped $d" | |
# continue | |
#fi | |
# Verify newest modification age requirement. | |
echo -e -n "\t${d} -> " | |
if ! [[ -e "${d}" ]]; then | |
echo 'FAILED - path does not exist' | |
elif ! [[ -d "${d}" ]]; then | |
echo 'FAILED - not a directory' | |
else | |
#age=$(${ageScript} "${d}" 2>/dev/null || :) | |
#if [[ -z "${age}" ]]; then | |
# echo 'FAILED - most recent modification not obtained' | |
#else | |
# ageInSeconds=$(echo "${age}" | cut -d' ' -f2) | |
# remaining=$((${minAgeInSecondsBeforeExtract}-${ageInSeconds})) | |
# if [[ ${remaining} -gt 0 ]]; then | |
# echo "FAILED - min age requirement will not be met for ${remaining} seconds" | |
# else | |
directories+=("${d}") | |
echo 'OK - path added' | |
# fi | |
#fi | |
fi | |
done | |
if [[ ${#directories[@]} -eq 0 ]]; then | |
echo 'error: no valid or readable directories found' 1>&2 | |
exit 1 | |
fi | |
echo -e "\nFound ${#directories[@]} directories\n" | |
} | |
function extractAndRemoveFromDirectories() { | |
for d in "${directories[@]}"; do | |
IFS_BAK="${IFS}" | |
IFS=$'\n' | |
# Extract RAR archives. | |
for subPath in $(find "${d}" -name '*.rar' | while read x; do echo $(dirname "${x}"); done | sort | uniq); do | |
#echo "subPath=\"${subPath}\"" | |
if [[ -z "$(find "${subPath}" -type d -maxdepth 0 -mtime +${minAgeInDays} -mtime -${maxAgeInDays})" ]] ; then | |
#echo "info: skipped ${subPath} due to age requirements not being met (too new or too old)" | |
continue | |
fi | |
age=$(${ageScript} "${subPath}" 2>/dev/null || :) | |
if [[ -z "${age}" ]]; then | |
echo "error: no age found for subPath=\"${subPath}\", path skipped" | |
continue | |
else | |
ageInSeconds=$(echo "${age}" | cut -d' ' -f2) | |
remaining=$((${minAgeInSecondsBeforeExtract}-${ageInSeconds})) | |
if [[ ${remaining} -gt 0 ]]; then | |
echo "info: skipped ${subPath}; need to wait ${remaining} more seconds" | |
continue | |
fi | |
fi | |
echo "Extracting ${subPath}" | |
cd "${subPath}" | |
# Temporarily rename *.!ut files. | |
overrideUtFileNames | |
# Locate RAR file. | |
rarFile="$(find . -name '*.rar' | head -n1)" | |
set +e | |
unrar t "${rarFile}" | |
rc=$? | |
set -e | |
if [[ "${rc}" = '0' ]]; then | |
set +e | |
unrar x -o+ "${rarFile}" | |
rc=$? | |
set -e | |
if [[ "${rc}" = '0' ]]; then | |
echo "info: deleting the following files: $(ls -1 | grep '\.\(rar\|r[0-9]\+\)$' | tr '\n' ':' | sed 's/:/, /g' | sed 's/, $//')" | |
for f in $(ls -1 | grep '\.\(rar\|r[0-9]\+\)$'); do | |
rm -f "${f}" | |
done | |
rm -rf Sample | |
echo "info: attempting de-queue from deluge" | |
set +e | |
set -x | |
/share/CACHEDEV1_DATA/homes/tddl deluge remove --name "$(basename $(pwd))" | |
rc=$? | |
set +x | |
set -e | |
if [[ "${rc}" != '0' ]]; then | |
echo "error: tddl returned non-zero exit status code=${rc}" | |
fi | |
else | |
restoreUtFileNames | |
echo "error: extraction failed for subPath=${subPath} rarFile=${rarFile}, rc=${rc}" 2>&2 | |
fi | |
else | |
restoreUtFileNames | |
echo "error: verification check failed for subPath=${subPath} rarFile=${rarFile}, rc=${rc}" 1>&2 | |
fi | |
done | |
# Remove uTorrent.dat files. | |
for datFile in $(find "${d}" -name '~uTorrentPartFile_*.dat'); do | |
rm -f "${datFile}" | |
done | |
IFS="${IFS_BAK}" | |
unset IFS_BAK | |
done | |
} | |
function main() { | |
basePath="$(pwd)" | |
cd "$(dirname "$0")" | |
collectEligibleDirectories $* | |
extractAndRemoveFromDirectories | |
exit 0 | |
} | |
# Validate parameters and environmental requirements. | |
if [[ -z "$*" ]]; then | |
echo -e '\nerror: missing required argument: directory name\n' 1>&2 | |
usage | |
fi | |
if [[ -z "$(which unrar)" ]]; then | |
echo -e '\nerror: missing `unrar` command-line program\n' 1>&2 | |
exit 1 | |
fi | |
main $* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# -*- coding: utf-8 -*- | |
""" | |
Returns a triple of (edit epoch time, age in seconds, filepath) to the most | |
recently modified file in a directory. | |
""" | |
import calendar | |
import os | |
import sys | |
import time | |
def find_most_recently_modified(path): | |
max_mtime = None | |
max_file = '' | |
for dirname, subdirs, files in os.walk(path): | |
for file in files: | |
full_path = os.path.join(dirname, file) | |
mtime = os.stat(full_path).st_mtime | |
if max_mtime is None or mtime > max_mtime: | |
max_mtime = mtime | |
max_file = full_path | |
for subdir in subdirs: | |
mtime, file = find_most_recently_modified(os.path.join(path, subdir)) | |
if max_mtime is None or mtime > max_mtime: | |
max_mtime = mtime | |
max_file = file | |
if max_mtime is None: | |
return -1, '' | |
return max_mtime, max_file | |
if __name__ == '__main__': | |
if len(sys.argv) < 2: | |
path = '.' | |
#sys.stderr.write('missing required parameter: dir-path\n') | |
#sys.exit(1) | |
else: | |
path = sys.argv[1] | |
mtime, file = find_most_recently_modified(path) | |
if mtime == -1: | |
sys.stderr.write('error: no results\n') | |
sys.exit(1) | |
age = int(calendar.timegm(time.gmtime()) - mtime) | |
print('%s %s %s' % (int(mtime), age, file)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
## | |
# | |
# @author Jay Taylor [@jtaylor] | |
# | |
# @date | |
# | |
# @description Busy-box compatible automatic RAR extraction system. | |
# | |
# Originally created on 2014-12-20. | |
# | |
# Updated 2020-03-18, now passes shellcheck but not yet tested. | |
# | |
set -o errexit | |
set -o pipefail | |
set -o nounset | |
ts() { | |
date -u +'%Y-%m-%dT%H:%M:%S%z' | |
} | |
log_debug() { | |
echo -e "$*" | sed "s/^./$(ts) DEBUG: &/" 1>&2 | |
} | |
log_info() { | |
echo -e "$*" | sed "s/^./$(ts) INFO: &/" 1>&2 | |
} | |
log_warn() { | |
echo -e "$*" | sed "s/^./$(ts) WARN: &/" 1>&2 | |
} | |
log_error() { | |
echo -e "$*" | sed "s/^./$(ts) ERROR: &/" 1>&2 | |
} | |
die() { | |
echo -e "$*" | sed "s/^./$(ts) FATAL: &/" 1>&2 | |
exit 1 | |
} | |
if [ "${1:-}" = '-v' ]; then | |
echo 'DEBUG: verbose mode enabled' 1>&2 | |
set -o xtrace | |
shift | |
fi | |
## | |
# Begin Configuration | |
# | |
# Set to 1 to enable de-queuing from remote deluge. | |
# n.b. requires 'tddl' cli program. | |
DEQUEUE_EXTRACTED_FROM_DELUGE=0 | |
min_age_in_days=3 | |
max_age_in_days=180 | |
# NB: 3 days worth of seconds = 259200. | |
min_age_in_seconds_before_extract="$((min_age_in_days*60*60*24))" | |
# | |
# End Configuration | |
## | |
function usage() { | |
echo "USAGE: $0 [directory-name..]" | |
exit 1 | |
} | |
#override_ut_file_names() { | |
# local ut_files | |
# ut_files="$(find . -name '*.\!ut')" | |
# if [ "$(echo "${ut_files}" | tr ' ' $'\n' | grep -v '^$' | wc -l)" -lt 10 ]; then | |
# for f in ${ut_files}; do | |
# mv "${f}" "$(echo "${f}" | sed 's/\.\!ut$//')" | |
# done | |
# fi | |
#} | |
#restore_ut_file_names() { | |
# local ut_files | |
# ut_files="$(find . -name '*.\!ut')" | |
# if [ "$(echo "${ut_files}" | tr ' ' $'\n' | grep -v '^$' | wc -l)" -lt 10 ]; then | |
# for f in ${ut_files}; do | |
# mv "$(echo "${f}" | sed 's/\.\!ut$//')" "${f}" | |
# done | |
# fi | |
#} | |
collect_eligible_directories() { | |
local directories=() | |
echo -e '\nINFO: Verifying specified directories:' 1>&2 | |
for d in "$@"; do | |
if ! echo "${d}" | grep -q '^\/'; then | |
d="${base_path}/${d}" | |
fi | |
#if [ -z "$(find "${d}" -type d -maxdepth 0 -mtime +${min_age_in_days} -mtime -${max_age_in_days})" ]; then | |
# echo "skipped $d" | |
# continue | |
#fi | |
# Verify newest modification age requirement. | |
echo -e -n "\t${d} -> " 1>&2 | |
if ! [ -e "${d}" ]; then | |
echo 'FAILED - path does not exist' 1>&2 | |
elif ! [ -d "${d}" ]; then | |
echo 'FAILED - not a directory' 1>&2 | |
else | |
#age=$(most-recently-modified.py "${d}" 2>/dev/null || :) | |
#if [[ -z "${age}" ]]; then | |
# echo 'FAILED - most recent modification not obtained' | |
#else | |
# age_in_seconds=$(echo "${age}" | cut -d' ' -f2) | |
# remaining=$((${min_age_in_seconds_before_extract}-${age_in_seconds})) | |
# if [[ ${remaining} -gt 0 ]]; then | |
# echo "FAILED - min age requirement will not be met for ${remaining} seconds" | |
# else | |
directories+=("${d}") | |
echo 'OK - path added' 1>&2 | |
# fi | |
#fi | |
fi | |
done | |
if [ "${#directories[@]}" -eq 0 ]; then | |
die 'no valid or readable directories found' | |
fi | |
echo -e "\nFound ${#directories[@]} directories\n" 1>&2 | |
} | |
extract_and_remove_from_directories() { | |
local ifs_bak | |
local x | |
local sub_path | |
local age | |
local age_in_seconds | |
local remaining | |
local rar_file | |
local rc | |
#local dat_file | |
for d in "${directories[@]}"; do | |
ifs_bak="${IFS}" | |
IFS=$'\n' | |
# Extract RAR archives. | |
find "${d}" -name '*.rar' | while read -r x; do | |
dirname "${x}" | |
done \ | |
| sort \ | |
| uniq \ | |
| while read -r -d $'\0' sub_path; do | |
#echo "sub_path=\"${sub_path}\"" | |
if [ -z "$(find "${sub_path}" -type d -maxdepth 0 -mtime +${min_age_in_days} -mtime -${max_age_in_days})" ]; then | |
#log_info "skipped ${sub_path} due to age requirements not being met (too new or too old)" | |
continue | |
fi | |
age="$(most-recently-modified.py "${sub_path}" 2>/dev/null || :)" | |
if [ -z "${age}" ]; then | |
log_warn "no age found for sub_path=\"${sub_path}\", path skipped" | |
continue | |
else | |
age_in_seconds="$(echo "${age}" | cut -d' ' -f2)" | |
remaining="$((min_age_in_seconds_before_extract-age_in_seconds))" | |
if [ "${remaining}" -gt 0 ]; then | |
log_info "skipped ${sub_path}; need to wait ${remaining} more seconds" | |
continue | |
fi | |
fi | |
log_info "Extracting ${sub_path}" | |
cd "${sub_path}" | |
# Temporarily rename *.!ut files. | |
#override_ut_file_names | |
# Locate RAR file. | |
rar_file="$(find . -name '*.rar' | head -n1)" | |
set +e | |
unrar t "${rar_file}" | |
rc=$? | |
set -e | |
if [ "${rc}" -eq 0 ]; then | |
set +o errexit | |
unrar x -o+ "${rar_file}" | |
rc=$? | |
set -o errexit | |
if [ "${rc}" = '0' ]; then | |
log_info "deleting the following files: $(find . -maxdepth | grep '\.\(rar\|r[0-9]\+\)$' | tr '\n' ':' | sed 's/:/, /g' | sed 's/, $//')" | |
for f in $(find . -maxdepth 1 | grep '\.\(rar\|r[0-9]\+\)$'); do | |
rm -f "${f}" | |
done | |
rm -rf 'Sample' 'sample' | |
if [ "${DEQUEUE_EXTRACTED_FROM_DELUGE:-0}" -ne 0 ]; then | |
log_info 'attempting de-queue from deluge' | |
set +o errexit | |
#set -x | |
tddl deluge remove --name "$(basename "$(pwd)")" | |
rc=$? | |
#set +x | |
set -o errexit | |
if [ "${rc}" != '0' ]; then | |
log_warn "tddl returned non-zero exit status code=${rc}" | |
fi | |
fi | |
else | |
#restore_ut_file_names | |
log_warn "extraction failed for sub_path=${sub_path} rar_file=${rar_file}, rc=${rc}" | |
fi | |
else | |
#restore_ut_file_names | |
log_warn "verification check failed for sub_path=${sub_path} rar_file=${rar_file}, rc=${rc}" | |
fi | |
done | |
## Remove uTorrent.dat files. | |
#find "${d}" -name '~uTorrentPartFile_*.dat' | while read -r -d $'\0' dat_file; do | |
# rm -f "${dat_file}" | |
#done | |
IFS="${ifs_bak}" | |
done | |
} | |
main() { | |
local base_path | |
base_path="$(pwd)" | |
# Validate parameters and environmental requirements. | |
if [ -z "$*" ] || [ "${1:-}" = '-h' ] || [ "${1:-}" = '--help' ]; then | |
log_error '\nmissing required argument: directory name\n' | |
usage | |
return 0 | |
fi | |
if ! command -v 'unrar' 1>/dev/null; then | |
die '\nmissing required program: "unrar"\n' | |
return 1 | |
fi | |
if [ "${DEQUEUE_EXTRACTED_FROM_DELUGE:-0}" -ne 0 ] && ! command -v 'tddl' 1>/dev/null; then | |
die '\nmissing required program: "tddl"\n' | |
return 1 | |
fi | |
cd "$(dirname "$0")" | |
collect_eligible_directories "$@" | |
extract_and_remove_from_directories | |
return 0 | |
} | |
if [ "${BASH_SOURCE[0]}" = "${0}" ] || [ "${BASH_SOURCE[0]}" = '--' ]; then | |
main "$@" | |
fi |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment