Skip to content

Instantly share code, notes, and snippets.

@jaytaylor
Last active March 18, 2020 18:06
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jaytaylor/16b6f5a7fa2e2e88b46c to your computer and use it in GitHub Desktop.
Save jaytaylor/16b6f5a7fa2e2e88b46c to your computer and use it in GitHub Desktop.
Busy-box compatible automatic RAR extraction system.
#!/usr/bin/env bash
##
#
# @author Jay Taylor [@jtaylor]
#
# @date 2014-12-20
#
# @description Busy-box compatible automatic RAR extraction system.
#
set -e
##
# Begin Configuration
#
minAgeInDays=3
maxAgeInDays=180
ageScript='/share/CACHEDEV1_DATA/homes/most-recently-modified.py'
# NB: 3 days worth of seconds = 259200.
minAgeInSecondsBeforeExtract=$((${minAgeInDays}*60*60*24))
#
# End Configuration
##
function usage() {
echo "usage: $0 [directory-name] [?directory-name]..." 1>&2
exit 1
}
function abortIfNonZero() {
# @param $1 command return code/exit status (e.g. $?, '0', '1').
# @param $2 error message if exit status was non-zero.
local rc=$1
local what=$2
test $rc -ne 0 && echo "error: ${what} exited with non-zero status ${rc}" 1>&2 && exit $rc || :
}
function overrideUtFileNames() {
utFiles="$(find . -name '*.\!ut')"
if [[ $(echo "${utFiles}" | tr ' ' $'\n' | grep -v '^$' | wc -l) -lt 10 ]]; then
for f in ${utFiles}; do
mv "${f}" "$(echo "${f}" | sed 's/\.\!ut$//')"
done
fi
}
function restoreUtFileNames() {
if [[ $(echo "${utFiles}" | tr ' ' $'\n' | grep -v '^$' | wc -l) -lt 10 ]]; then
for f in ${utFiles}; do
mv "$(echo "${f}" | sed 's/\.\!ut$//')" "${f}"
done
fi
}
function collectEligibleDirectories() {
directories=()
echo -e '\nVerifying specified directories:'
for d in $*; do
if [[ -z "$(echo "${d}" | grep '^\/' )" ]]; then
d="${basePath}/${d}"
fi
#if [[ -z "$(find "${d}" -type d -maxdepth 0 -mtime +${minAgeInDays} -mtime -${maxAgeInDays})" ]] ; then
# echo "skipped $d"
# continue
#fi
# Verify newest modification age requirement.
echo -e -n "\t${d} -> "
if ! [[ -e "${d}" ]]; then
echo 'FAILED - path does not exist'
elif ! [[ -d "${d}" ]]; then
echo 'FAILED - not a directory'
else
#age=$(${ageScript} "${d}" 2>/dev/null || :)
#if [[ -z "${age}" ]]; then
# echo 'FAILED - most recent modification not obtained'
#else
# ageInSeconds=$(echo "${age}" | cut -d' ' -f2)
# remaining=$((${minAgeInSecondsBeforeExtract}-${ageInSeconds}))
# if [[ ${remaining} -gt 0 ]]; then
# echo "FAILED - min age requirement will not be met for ${remaining} seconds"
# else
directories+=("${d}")
echo 'OK - path added'
# fi
#fi
fi
done
if [[ ${#directories[@]} -eq 0 ]]; then
echo 'error: no valid or readable directories found' 1>&2
exit 1
fi
echo -e "\nFound ${#directories[@]} directories\n"
}
function extractAndRemoveFromDirectories() {
for d in "${directories[@]}"; do
IFS_BAK="${IFS}"
IFS=$'\n'
# Extract RAR archives.
for subPath in $(find "${d}" -name '*.rar' | while read x; do echo $(dirname "${x}"); done | sort | uniq); do
#echo "subPath=\"${subPath}\""
if [[ -z "$(find "${subPath}" -type d -maxdepth 0 -mtime +${minAgeInDays} -mtime -${maxAgeInDays})" ]] ; then
#echo "info: skipped ${subPath} due to age requirements not being met (too new or too old)"
continue
fi
age=$(${ageScript} "${subPath}" 2>/dev/null || :)
if [[ -z "${age}" ]]; then
echo "error: no age found for subPath=\"${subPath}\", path skipped"
continue
else
ageInSeconds=$(echo "${age}" | cut -d' ' -f2)
remaining=$((${minAgeInSecondsBeforeExtract}-${ageInSeconds}))
if [[ ${remaining} -gt 0 ]]; then
echo "info: skipped ${subPath}; need to wait ${remaining} more seconds"
continue
fi
fi
echo "Extracting ${subPath}"
cd "${subPath}"
# Temporarily rename *.!ut files.
overrideUtFileNames
# Locate RAR file.
rarFile="$(find . -name '*.rar' | head -n1)"
set +e
unrar t "${rarFile}"
rc=$?
set -e
if [[ "${rc}" = '0' ]]; then
set +e
unrar x -o+ "${rarFile}"
rc=$?
set -e
if [[ "${rc}" = '0' ]]; then
echo "info: deleting the following files: $(ls -1 | grep '\.\(rar\|r[0-9]\+\)$' | tr '\n' ':' | sed 's/:/, /g' | sed 's/, $//')"
for f in $(ls -1 | grep '\.\(rar\|r[0-9]\+\)$'); do
rm -f "${f}"
done
rm -rf Sample
echo "info: attempting de-queue from deluge"
set +e
set -x
/share/CACHEDEV1_DATA/homes/tddl deluge remove --name "$(basename $(pwd))"
rc=$?
set +x
set -e
if [[ "${rc}" != '0' ]]; then
echo "error: tddl returned non-zero exit status code=${rc}"
fi
else
restoreUtFileNames
echo "error: extraction failed for subPath=${subPath} rarFile=${rarFile}, rc=${rc}" 2>&2
fi
else
restoreUtFileNames
echo "error: verification check failed for subPath=${subPath} rarFile=${rarFile}, rc=${rc}" 1>&2
fi
done
# Remove uTorrent.dat files.
for datFile in $(find "${d}" -name '~uTorrentPartFile_*.dat'); do
rm -f "${datFile}"
done
IFS="${IFS_BAK}"
unset IFS_BAK
done
}
function main() {
basePath="$(pwd)"
cd "$(dirname "$0")"
collectEligibleDirectories $*
extractAndRemoveFromDirectories
exit 0
}
# Validate parameters and environmental requirements.
if [[ -z "$*" ]]; then
echo -e '\nerror: missing required argument: directory name\n' 1>&2
usage
fi
if [[ -z "$(which unrar)" ]]; then
echo -e '\nerror: missing `unrar` command-line program\n' 1>&2
exit 1
fi
main $*
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Returns a triple of (edit epoch time, age in seconds, filepath) to the most
recently modified file in a directory.
"""
import calendar
import os
import sys
import time
def find_most_recently_modified(path):
max_mtime = None
max_file = ''
for dirname, subdirs, files in os.walk(path):
for file in files:
full_path = os.path.join(dirname, file)
mtime = os.stat(full_path).st_mtime
if max_mtime is None or mtime > max_mtime:
max_mtime = mtime
max_file = full_path
for subdir in subdirs:
mtime, file = find_most_recently_modified(os.path.join(path, subdir))
if max_mtime is None or mtime > max_mtime:
max_mtime = mtime
max_file = file
if max_mtime is None:
return -1, ''
return max_mtime, max_file
if __name__ == '__main__':
if len(sys.argv) < 2:
path = '.'
#sys.stderr.write('missing required parameter: dir-path\n')
#sys.exit(1)
else:
path = sys.argv[1]
mtime, file = find_most_recently_modified(path)
if mtime == -1:
sys.stderr.write('error: no results\n')
sys.exit(1)
age = int(calendar.timegm(time.gmtime()) - mtime)
print('%s %s %s' % (int(mtime), age, file))
#!/usr/bin/env bash
##
#
# @author Jay Taylor [@jtaylor]
#
# @date
#
# @description Busy-box compatible automatic RAR extraction system.
#
# Originally created on 2014-12-20.
#
# Updated 2020-03-18, now passes shellcheck but not yet tested.
#
set -o errexit
set -o pipefail
set -o nounset
ts() {
date -u +'%Y-%m-%dT%H:%M:%S%z'
}
log_debug() {
echo -e "$*" | sed "s/^./$(ts) DEBUG: &/" 1>&2
}
log_info() {
echo -e "$*" | sed "s/^./$(ts) INFO: &/" 1>&2
}
log_warn() {
echo -e "$*" | sed "s/^./$(ts) WARN: &/" 1>&2
}
log_error() {
echo -e "$*" | sed "s/^./$(ts) ERROR: &/" 1>&2
}
die() {
echo -e "$*" | sed "s/^./$(ts) FATAL: &/" 1>&2
exit 1
}
if [ "${1:-}" = '-v' ]; then
echo 'DEBUG: verbose mode enabled' 1>&2
set -o xtrace
shift
fi
##
# Begin Configuration
#
# Set to 1 to enable de-queuing from remote deluge.
# n.b. requires 'tddl' cli program.
DEQUEUE_EXTRACTED_FROM_DELUGE=0
min_age_in_days=3
max_age_in_days=180
# NB: 3 days worth of seconds = 259200.
min_age_in_seconds_before_extract="$((min_age_in_days*60*60*24))"
#
# End Configuration
##
function usage() {
echo "USAGE: $0 [directory-name..]"
exit 1
}
#override_ut_file_names() {
# local ut_files
# ut_files="$(find . -name '*.\!ut')"
# if [ "$(echo "${ut_files}" | tr ' ' $'\n' | grep -v '^$' | wc -l)" -lt 10 ]; then
# for f in ${ut_files}; do
# mv "${f}" "$(echo "${f}" | sed 's/\.\!ut$//')"
# done
# fi
#}
#restore_ut_file_names() {
# local ut_files
# ut_files="$(find . -name '*.\!ut')"
# if [ "$(echo "${ut_files}" | tr ' ' $'\n' | grep -v '^$' | wc -l)" -lt 10 ]; then
# for f in ${ut_files}; do
# mv "$(echo "${f}" | sed 's/\.\!ut$//')" "${f}"
# done
# fi
#}
collect_eligible_directories() {
local directories=()
echo -e '\nINFO: Verifying specified directories:' 1>&2
for d in "$@"; do
if ! echo "${d}" | grep -q '^\/'; then
d="${base_path}/${d}"
fi
#if [ -z "$(find "${d}" -type d -maxdepth 0 -mtime +${min_age_in_days} -mtime -${max_age_in_days})" ]; then
# echo "skipped $d"
# continue
#fi
# Verify newest modification age requirement.
echo -e -n "\t${d} -> " 1>&2
if ! [ -e "${d}" ]; then
echo 'FAILED - path does not exist' 1>&2
elif ! [ -d "${d}" ]; then
echo 'FAILED - not a directory' 1>&2
else
#age=$(most-recently-modified.py "${d}" 2>/dev/null || :)
#if [[ -z "${age}" ]]; then
# echo 'FAILED - most recent modification not obtained'
#else
# age_in_seconds=$(echo "${age}" | cut -d' ' -f2)
# remaining=$((${min_age_in_seconds_before_extract}-${age_in_seconds}))
# if [[ ${remaining} -gt 0 ]]; then
# echo "FAILED - min age requirement will not be met for ${remaining} seconds"
# else
directories+=("${d}")
echo 'OK - path added' 1>&2
# fi
#fi
fi
done
if [ "${#directories[@]}" -eq 0 ]; then
die 'no valid or readable directories found'
fi
echo -e "\nFound ${#directories[@]} directories\n" 1>&2
}
extract_and_remove_from_directories() {
local ifs_bak
local x
local sub_path
local age
local age_in_seconds
local remaining
local rar_file
local rc
#local dat_file
for d in "${directories[@]}"; do
ifs_bak="${IFS}"
IFS=$'\n'
# Extract RAR archives.
find "${d}" -name '*.rar' | while read -r x; do
dirname "${x}"
done \
| sort \
| uniq \
| while read -r -d $'\0' sub_path; do
#echo "sub_path=\"${sub_path}\""
if [ -z "$(find "${sub_path}" -type d -maxdepth 0 -mtime +${min_age_in_days} -mtime -${max_age_in_days})" ]; then
#log_info "skipped ${sub_path} due to age requirements not being met (too new or too old)"
continue
fi
age="$(most-recently-modified.py "${sub_path}" 2>/dev/null || :)"
if [ -z "${age}" ]; then
log_warn "no age found for sub_path=\"${sub_path}\", path skipped"
continue
else
age_in_seconds="$(echo "${age}" | cut -d' ' -f2)"
remaining="$((min_age_in_seconds_before_extract-age_in_seconds))"
if [ "${remaining}" -gt 0 ]; then
log_info "skipped ${sub_path}; need to wait ${remaining} more seconds"
continue
fi
fi
log_info "Extracting ${sub_path}"
cd "${sub_path}"
# Temporarily rename *.!ut files.
#override_ut_file_names
# Locate RAR file.
rar_file="$(find . -name '*.rar' | head -n1)"
set +e
unrar t "${rar_file}"
rc=$?
set -e
if [ "${rc}" -eq 0 ]; then
set +o errexit
unrar x -o+ "${rar_file}"
rc=$?
set -o errexit
if [ "${rc}" = '0' ]; then
log_info "deleting the following files: $(find . -maxdepth | grep '\.\(rar\|r[0-9]\+\)$' | tr '\n' ':' | sed 's/:/, /g' | sed 's/, $//')"
for f in $(find . -maxdepth 1 | grep '\.\(rar\|r[0-9]\+\)$'); do
rm -f "${f}"
done
rm -rf 'Sample' 'sample'
if [ "${DEQUEUE_EXTRACTED_FROM_DELUGE:-0}" -ne 0 ]; then
log_info 'attempting de-queue from deluge'
set +o errexit
#set -x
tddl deluge remove --name "$(basename "$(pwd)")"
rc=$?
#set +x
set -o errexit
if [ "${rc}" != '0' ]; then
log_warn "tddl returned non-zero exit status code=${rc}"
fi
fi
else
#restore_ut_file_names
log_warn "extraction failed for sub_path=${sub_path} rar_file=${rar_file}, rc=${rc}"
fi
else
#restore_ut_file_names
log_warn "verification check failed for sub_path=${sub_path} rar_file=${rar_file}, rc=${rc}"
fi
done
## Remove uTorrent.dat files.
#find "${d}" -name '~uTorrentPartFile_*.dat' | while read -r -d $'\0' dat_file; do
# rm -f "${dat_file}"
#done
IFS="${ifs_bak}"
done
}
main() {
local base_path
base_path="$(pwd)"
# Validate parameters and environmental requirements.
if [ -z "$*" ] || [ "${1:-}" = '-h' ] || [ "${1:-}" = '--help' ]; then
log_error '\nmissing required argument: directory name\n'
usage
return 0
fi
if ! command -v 'unrar' 1>/dev/null; then
die '\nmissing required program: "unrar"\n'
return 1
fi
if [ "${DEQUEUE_EXTRACTED_FROM_DELUGE:-0}" -ne 0 ] && ! command -v 'tddl' 1>/dev/null; then
die '\nmissing required program: "tddl"\n'
return 1
fi
cd "$(dirname "$0")"
collect_eligible_directories "$@"
extract_and_remove_from_directories
return 0
}
if [ "${BASH_SOURCE[0]}" = "${0}" ] || [ "${BASH_SOURCE[0]}" = '--' ]; then
main "$@"
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment