pkg_mirror.sh

#!/bin/ksh

get_manifests()
{
	repo="$1"
	target="$2"

	saved_cwd="$PWD"
	cd "$target"

	wget -q -O - "${repo}/catalog/0" | awk '$1 == "V" { print $2; }' | \
	    sed -e "s,/,%2F,g; s,^pkg:%2F,${repo}/manifest/0/," | \
	    xargs wget -q -nc --continue
	for manifest in $(echo *); do
		pkg=$(echo "$manifest" | \
		    sed -e 's,@,/,g; s/,/%2C/g; s,:,%3A,g')
		mkdir -p "../${pkg}"
		cp -p "$manifest" "../${pkg}/manifest"
	done
	cd "$saved_cwd"
}

get_files()
{ 
	repo="$1"
	manifest="$2"
	dir="$3"
	pkg="$4"

	saved_cwd="$PWD"
	cd "$dir"
	for file in $(awk \
	    '($1 == "file" || $1 == "license") { print $2; }' "$manifest" | \
	    sort -u); do
		parent=$(echo "$file" | sed 's,\(..\)\(..\).*,\1/\2,')
		mkdir -p "$parent"
		if [ -f "$file" ]; then
			mv "$file" "${parent}/${file}"
		fi
		if [ ! -L "${pkg}/${file}" ]; then
			ln -s "${PWD}/${parent}/${file}" "${pkg}/${file}"
		fi
		if [ -f "${parent}/${file}" ]; then
			# File already exists, better to do some I/O than to
			# download it again
			hash=$(digest -a sha1 "${parent}/${file}")
			if [ "$hash" = "$file" ]; then
				continue
			fi

		fi
		wget -q -O - "${repo}/file/0/${file}" | \
		    gunzip -c > "${parent}/${file}"
	done

	cd "$saved_cwd"
}

get_pkgs()
{
	repo="$1"
	target="$2"

	saved_cwd="$PWD"
	cd "$target"

	mkdir -p "manifests"
	get_manifests "$repo" "manifests"

	mkdir -p "files"
	for manifest in $(echo */*/manifest); do
		echo "Fetching files for ${manifest}..."
		get_files "$repo" "${PWD}/${manifest}" "files" "$PWD"
	done

	cd "$saved_cwd"
}

mirror_repo()
{
	repo="$1"
	target="$2"

	reponame=$(echo "$repo" | sed 's,.*/,,g')
	mkdir -p "$reponame"
	get_pkgs "$repo" "$reponame"
	publish_pkgs "$reponame" "$target"
}

publish_pkgs()
{
	from="$1"
	target="$2"

	for pkg in $(echo $from/*/*); do
		[ -d "$pkg" ] || continue
		pkgsend -s "$target" publish --fmri-in-manifest --no-catalog \
		   --no-index -d "$pkg" -d "${from}/files" "${pkg}/manifest"
	done
}

for repo in $@; do
	reponame=$(echo "$repo" | sed 's,.*/,,g')
	mirror_repo "$repo" "../${reponame}"
done