Skip to content

Instantly share code, notes, and snippets.

@kergoth
Created October 8, 2009 18:42
Show Gist options
  • Save kergoth/205260 to your computer and use it in GitHub Desktop.
Save kergoth/205260 to your computer and use it in GitHub Desktop.
"Magic" rsync
#!/bin/bash
shopt -s compat31 2>/dev/null
usage () {
echo 'Usage: magic_rsync.sh [OPTION]... FROM TO'
echo 'Run rsync from one location to another, manually processing specific groups of files.'
echo
echo ' -p PAT -c CMD For each file matching pattern PAT, run CMD against it.'
echo ' CMD is passed the filename relative to transfer root, FROM, and TO.'
echo ' -x EXCLUDEPAT Each exclude pattern is passed directly to rsync with --exclude.'
echo
echo 'Example:'
echo ' magic_rsync.sh -p "*.pdf" -c "copy_and_compress" -x "/Calibre/" -x "*.chm" Books Books-for-Reader'
echo ' This runs copy_and_compress to populate Books-for-reader with the pdfs, compressing them as'
echo ' appropriate for the Ebook reader. Other files, like .epub, will be rsyncd over directly.'
exit 2
}
abort () {
echo >&2 "User aborted."
exit 1
}
greatest_common_path () {
python -c 'import os, sys; print os.path.commonprefix(sys.argv[1:])' "$@"
}
abspath () {
for arg; do
testabs=${arg##[^/]*}
echo ${testabs:-$PWD/$arg}
done
}
_rsync () {
#echo >&2 "rsync -auvm --del " "$@"
rsync -auvm --del "$@"
}
runrsync () {
if [[ -z ${excludeargs[*]} ]]; then
for exclude in "${excludes[@]}"; do
excludeargs[${#excludeargs[*]}]="--exclude"
excludeargs[${#excludeargs[*]}]="$exclude"
done
fi
_rsync "${excludeargs[@]}" "$@" "$from/" "$to"
}
rsync_rest () {
if [[ -z ${patterns_excludeargs[*]} ]]; then
for pattern in "${patterns[@]}"; do
patterns_excludeargs[${#patterns_excludeargs[*]}]="--exclude"
patterns_excludeargs[${#patterns_excludeargs[*]}]="$pattern"
done
fi
runrsync "${patterns_excludeargs[@]}" "$@"
}
rsync_magic () {
if [[ -z ${patterns_includeargs[*]} ]]; then
for pattern in "${patterns[@]}"; do
patterns_includeargs[${#patterns_includeargs[*]}]="--include"
patterns_includeargs[${#patterns_includeargs[*]}]="$pattern"
done
fi
for ((patnum = 0; patnum < ${#patterns[*]}; patnum++)); do
pattern="${patterns[$patnum]}"
command="${commands[$patnum]}"
local -a otherpats
otherpats=()
for otherpat in "${patterns[@]}"; do
if [[ "$otherpat" != "$pattern" ]]; then
otherpats[${#otherpats[*]}]="--exclude"
otherpats[${#otherpats[*]}]="$otherpat"
fi
done
runrsync -ni --include "*/" "${otherpats[@]}" --include "$pattern" \
--exclude "*" | \
while read changes fn; do
destfn="$to/$fn"
destdir="${destfn%/*}"
if [[ "$changes" =~ ">" ]]; then
mkdir -p "$destdir"
$command "$from/$fn" "$destfn"
elif [[ "$changes" = "*deleting" ]]; then
if [[ ! "$fn" =~ "/$" ]]; then
rm -f "$destfn"
echo "Deleted '$destfn'"
fi
rmdir -p "$destdir" 2>/dev/null >&2
fi
done
done
}
trap abort INT
excludes=()
patterns=()
commands=()
while [[ $# -gt 0 ]]; do
case "$1" in
-p)
shift || usage
patterns[${#patterns[*]}]="$1"
;;
-c)
shift || usage
commands[${#patterns[*]}-1]="$1"
;;
-x)
shift || usage
excludes[${#excludes[*]}]="$1"
;;
*)
break
;;
esac
shift
done
if [[ ${#patterns[*]} -ne ${#commands[*]} ]]; then
echo >&2 "Error: mismatched -p and -c arguments. A -c argument must exist for each -p argument."
usage
fi
from="`abspath \"$1\"`"
shift || usage
to="`abspath \"$1\"`"
shift || usage
rsync_transfer_root="`greatest_common_path \"$from\" \"$to\"`"
rsync_transfer_root="${rsync_transfer_root%/}/"
excludes[${#excludes[*]}]="/${to#$rsync_transfer_root}"
rsync_rest "$@"
rsync_magic
#!/bin/bash
# Current workflow:
#
# Prior to running this:
# - Download an ebook.
# - If not pdf or epub, convert to pdf or epub.
# - Set pdf & epub metadata correctly.
# - Rename pdf and epub by their metadata.
# - Move the ebook into the appropriate location in ~/Books/
#
# Phase 1: Compress the PDFs using PDF Shrink
# (~/Books/ -> ~/Books/PRS-505 Staging/)
# Phase 2: Run sopdf against the pdfs to crop the pages,
# split them vertically, and rotate them horizontally.
# (~/Books/PRS-505 Staging/ -> ~/Books/PRS-505/)
# Phase 3: Sync to /Volumes/PRS-505 SD/)
#
# NOTES:
# - Directories with any finder label are excluded from the operation entirely
# - Individual PDF files:
# - Blue: Doesn't affect processing, but indicates that it isn't the original
# format, and that the original is still around.
# - Purple: Copies, but doesn't compress.
# - Green: Does not copy.
testabs=${0##[^/]*}
binpath=${testabs:-$PWD/$0}
bindir=${binpath%/*}
BOOKSPATH=$(cd "$bindir/../.." && pwd)
STAGING1="$BOOKSPATH/Reader Staging"
STAGING2="$BOOKSPATH/PRS-505 Staging"
shopt -s compat31 2>/dev/null
copy_and_compress () {
srcfn="$1"
destfn="$2"
flags="`mdls -name kMDItemFSFinderFlags \"$srcfn\" | sed -e's,^.* = ,,'`"
if [[ $flags -eq 4 ]]; then # Green
rm -f "$destfn"
return 0
fi
ditto "$srcfn" "$destfn" || {
echo >&2 "Warning: Unable to copy to '$destfn'"
return 1
}
echo "Copied '$destfn'"
if [[ $flags -eq 6 ]]; then # Purple
return 0
fi
open -Wga "Compress PDF for iPhone" "$destfn" || {
rm -f "$destfn"
echo >&2 "Warning: Unable to compress '$destfn'"
return 1
}
echo "Compressed '$destfn'"
return 0
}
# -m 3 == vertical
# -m 0 (default) == 2xhorizontal
sopdfopts="-e -m 3"
sopdf () {
if [[ $system = Linux ]]; then
$bindir/../sopdf-0.1a/sopdf -i "$1" $sopdfopts -o "$2"
else
srcfn="`cygpath -m \"$1\"`"
destfn="`cygpath -m \"$2\"`"
$bindir/../soPdf.exe -i "$srcfn" $sopdfopts -o "$destfn"
fi
exit 1
#if [[ $? -ne 0 ]]; then
# cp -pP "$1" "$2"
#fi
}
gs_calibre () {
bounding="`mktemp bounding.XXXXXX`"
gs -dSAFER -dNOPAUSE -dBATCH -sDEVICE#bbox "$1" 2>>$bounding || {
rm -f "$bounding"
echo >&2 "Unable to process file with ghostscript"
return 1
}
pdfmanipulate crop -o "$2" -b $bounding "$1" || {
rm -f "$bounding"
echo >&2 "Unable to adjust bounding boxes with pdfmanipulate"
return 1
}
rm -f $bounding
}
greatest_common_path () {
python -c 'import os, sys; common = os.path.commonprefix([p.split(os.sep) for p in sys.argv[1:]]); print(os.sep.join(common))' "$@"
}
abspath () {
for arg; do
testabs=${arg##[^/]*}
echo ${testabs:-$PWD/$arg}
done
}
normpath () {
for arg; do
cd "$(dirname "$arg")" && {
echo $(pwd)/$(basename $arg)
cd -
}
done
}
abort () {
echo >&2 "User aborted."
exit 1
}
trap abort INT
excludes=(-x .DS_Store)
excludes_from_flags () {
rsync_transfer_root=$(greatest_common_path "$@")
# All files or directories in rsync_transfer_root which are labeled in Finder are excluded automatically
obtain_excluded () {
cd $rsync_transfer_root
find . -maxdepth 2 -type d | cut -d/ -f2- | grep -v ".DS_Store" | while read path; do
flags="$(mdls -name kMDItemFSFinderFlags "$path" | sed -e's,^.* = ,,')" || continue
if [[ $flags -ne 0 ]]; then
echo "$path/"
fi
done
}
OIFS="$IFS"
IFS=$'\n'
for relpath in $(obtain_excluded); do
excludes[${#excludes[*]}]="-x"
excludes[${#excludes[*]}]="/$relpath"
done
IFS="$OIFS"
}
system="`uname -s`"
phase="$1"
shift
case "$phase" in
1)
from="$BOOKSPATH"
to="$STAGING1"
excludes_from_flags "$from"
. $bindir/magic_rsync.sh -p "*.pdf" -c copy_and_compress \
"${excludes[@]}" \
"$from" "$to" \
--include "*/" \
--exclude "*"
;;
2)
from="$STAGING1"
to="$STAGING2"
cmd="$1"
if [[ -z $cmd ]]; then
cmd="sopdf"
fi
. $bindir/magic_rsync.sh -p "*.pdf" -c "$cmd" \
"${excludes[@]}" \
"$from" "$to" \
--include "*/" \
--exclude "*"
;;
3)
from="$BOOKSPATH"
to="$(abspath "$1")"
shift || {
echo >&2 "Usage: $0 3 DESTPATH"
exit 2
}
excludes_from_flags "$from"
$bindir/magic_rsync.sh \
"$STAGING2" "$to" \
--modify-window=1 \
--include "*/" \
--exclude "._*" \
--include "*.pdf" \
--exclude "*"
$bindir/magic_rsync.sh \
"${excludes[@]}" \
"$from" "$to" \
--modify-window=1 \
--include "*/" \
--exclude "._*" \
--include "*.epub" --include "*.rtf" --include "*.txt" \
--include "*.lrf" --include "*.lrx" \
--exclude "*"
;;
4)
from="$BOOKSPATH"
to="$(abspath "$1")"
shift || {
echo >&2 "Usage: $0 4 DESTPATH"
exit 2
}
excludes_from_flags "$from"
$bindir/magic_rsync.sh \
"${excludes[@]}" \
"$from" "$to" \
--modify-window=1 \
--include "*/" \
--exclude "._*" \
--include "*.pdf" \
--include "*.epub" --include "*.rtf" --include "*.txt" \
--exclude "*"
;;
5)
from="$BOOKSPATH"
to="$(abspath "$1")"
shift || {
echo >&2 "Usage: $0 5 DESTPATH"
exit 2
}
excludes_from_flags "$from"
$bindir/magic_rsync.sh \
"${excludes[@]}" \
"$from" "$to" \
--modify-window=1 \
--include "*/" \
--exclude "._*" \
--include "*.epub" --include "*.rtf" --include "*.txt" \
--include "*.lrf" --include "*.lrx" \
--exclude "*"
;;
esac
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment