Download big git repositories over HTTP (using curl now, might want to replace with corresponding wget...)

download-git-by-http.sh

#!/bin/bash
link=$1
to=$2

echo "Link $link"
echo "To $2"

function createdirs {
  echo "createdirs Arg $1"
  for dir in $(curl $link$1 | grep "href=" | sed 's/^.*href="//' | sed 's/">.*$//' | grep -v '/git' | grep -v '^\?' | grep '/$')
  do mkdir -p $to$1$dir
   echo "To arg dir $to$1$dir"
   createdirs $1$dir
   echo "recursing $1$dir"
  done
  getfiles $1
}

function getfiles {
  echo "getfiles Arg $1"
  for file in $(curl $link$1 | grep "href=" | sed 's/^.*href="//' | sed 's/">.*$//' | grep -v '/git' | grep -v '^\?' | grep -v '/$')
  do curl $link$1$file -o $to$1$file
   echo "Link arg file $link$1$file to arg file $to$1$file"
  done
}

createdirs /