Creates Rubygems mirror in the current directory using Bash, some ruby and wget

rubygems_mirror.sh

#!/bin/bash
DEFAULT_URI='http://production.cf.rubygems.org/'
GEM_MARSHAL_VERSION=`ruby -e 'puts Gem.marshal_version'`
SPECS_FILE_Z="specs.$GEM_MARSHAL_VERSION.gz"
AVAILABLE_GEMS=`mktemp /tmp/available_gems.XXX`
GEMS_ALREADY_DOWNLOADED=`mktemp /tmp/gems_already_downloaded.XXX`
GEMS_TO_DOWNLOAD=`mktemp /tmp/gems_to_download.XXX`
GEMS_TO_REMOVE=`mktemp /tmp/gems_to_remove.XXX`
PARALLEl_DOWNLOAD=15

# get new gem specs
wget -O - --progress=bar ${DEFAULT_URI}${SPECS_FILE_Z} | gunzip | ruby -rcsv -e '
Marshal.load(STDIN.read).each do |name,ver,plat|
  puts "#{name}-#{ver}#{"-#{plat}" unless plat == "ruby"}.gem"
end' | sort > $AVAILABLE_GEMS
num=$((`wc -l < $AVAILABLE_GEMS`))
echo "There are $num gems available."

# remove old gems (and files like *.gem.1 (created by curl/wget)
ls -1 | comm -23 - $AVAILABLE_GEMS | egrep '\.gem(\.[0-9]+)?$' > $GEMS_TO_REMOVE
num=$((`wc -l < $GEMS_TO_REMOVE`))
if (($num > 0)); then
	read -p "Do you want to remove $num old gems before downloading the missing ones? "
	if [[ $REPLY =~ 'y' ]]; then
		xargs rm < $GEMS_TO_REMOVE
		echo "Removed $num gems."
	fi
fi

# count how many gems are already downloaded
ls -1 | comm -12 - $AVAILABLE_GEMS > $GEMS_ALREADY_DOWNLOADED
num=$((`wc -l < $GEMS_ALREADY_DOWNLOADED`))
echo "There are $num gems that are already downloaded."

# generate list of missing gems
if [[ $1 = '--verify' ]]; then
	# verify downloaded gems (wget will skip many of them because of --timestamping)
	cp $GEMS_ALREADY_DOWNLOADED $GEMS_TO_DOWNLOAD
	echo "All gems already downloaded will be verified."
else
	ls -1 | comm -13 - $AVAILABLE_GEMS > $GEMS_TO_DOWNLOAD
	echo "Only missing gems will be downloaded."
fi

# count missing gems
num=$((`wc -l < $GEMS_TO_DOWNLOAD`))
read -p "About to download $num gems. Is that OK? "
[[ $REPLY =~ 'y' ]] || exit

# generate URL list
while read gem; do
  echo "${DEFAULT_URI}gems/${gem}"
done < $GEMS_TO_DOWNLOAD > $GEMS_TO_DOWNLOAD.url

# download
echo "Downloading with $PARALLEl_DOWNLOAD parallel processes ..."
xargs -P $PARALLEl_DOWNLOAD wget -qc --timestamping --input-file - < $GEMS_TO_DOWNLOAD.url

# clean up
rm -f $AVAILABLE_GEMS $GEMS_TO_DOWNLOAD $GEMS_TO_REMOVE $GEMS_ALREADY_DOWNLOADED