gistfile1.sh

#!/bin/bash
# Run like so: echo http://cam.hackerspace.sg.s3-ap-southeast-1.amazonaws.com/ | ./listing.sh

s3ns=http://s3.amazonaws.com/doc/2006-03-01/

while read s3url
do

	test "$s3url" || continue
	i=0

	# Normalise URL
	s3url=$(curl $s3url -s -L -I -o /dev/null -w '%{url_effective}')

	s3get=$s3url # s3get gets marker requests appended to it

	while :; do
		curl -f -s $s3get > "listing$i.xml"
		if test $? -ne 0
		then
			echo ERROR $? retrieving: $s3get 1>&2
			break
		fi
		#xml sel -N w="http://s3.amazonaws.com/doc/2006-03-01/" -T -t -m "//w:Key" -o "$s3url" -v . -n "listing$i.xml"
		nextkey=$(xml sel -T -N "w=$s3ns" -t \
			--if '/w:ListBucketResult/w:IsTruncated="true"' \
			-v 'str:encode-uri(/w:ListBucketResult/w:Contents[last()]/w:Key, true())' \
			-b -n "listing$i.xml")
		# -b -n adds a newline to the result unconditionally,
		# this avoids the "no XPaths matched" message; $() drops newlines.
		echo listing$i.xml 1>&2
		echo $nextkey 1>&2

		read -p "Press [Enter] key to continue..."

		rm -f "listing$i.xml"

		if [ -n "$nextkey" ] ; then
			s3get=$s3get"?marker=$nextkey"
			i=$((i+1))
		else
			break
		fi
	done

done