fetch_distfiles.sh

#!/bin/sh
#
# Created by Bjorn Nelson 051222
#
# Description: fetch distfiles (more up to date then mirroring freebsd mirrors)

# Set these variables
PORTSDIR="/usr/ports"
IGNOREFILES=". Tools work files Mk Templates distfiles"
SUBPROC=30
CONCURRENTFETCH=15

# File location variables
BASENAME_LOC="/usr/bin/basename"
DATE_LOC="/bin/date"
DIRNAME_LOC="/usr/bin/dirname"
GREP_LOC="/usr/bin/grep"
PS_LOC="/bin/ps"
SLEEP_LOC="/bin/sleep"
WC_LOC="/usr/bin/wc"

# disable dialogs and interactivity
#BATCH=1

echo "Distfile fetch started: `$DATE_LOC`"

ALLPORTS=`cd $PORTSDIR && find . -type d -depth 2 -print`
for PORT in $ALLPORTS; do
  PORTNAME=`$BASENAME_LOC ${PORT#./}` 
  DIRNAME=`$DIRNAME_LOC $PORT` 
  PORTCAT=`$BASENAME_LOC ${DIRNAME#./}` 
  SKIP=0
  for IGNORE in $IGNOREFILES; do
    if [ $PORTNAME = $IGNORE -o $PORTCAT = $IGNORE ]; then
      SKIP=1
      break
    fi
  done
  if [ $SKIP -eq 1 ]; then
    continue
  fi
  START_SUBPROC=1
  while [ $START_SUBPROC = 1 ]; do
    SUBPROC_NUM=`$PS_LOC -ax | $GREP_LOC 'make fetch' | $GREP_LOC -v 'grep' | $WC_LOC -l`
    CONCURRENTFETCH_NUM=`$PS_LOC -ax | $GREP_LOC 'fetch -ARr' | $GREP_LOC -v 'grep' | $WC_LOC -l`
    if [ $SUBPROC_NUM -le $SUBPROC -a $CONCURRENTFETCH_NUM -le $CONCURRENTFETCH ]; then
      echo "Fetching ${PORT#./}"
      BATCH=YES
      export BATCH
      cd $PORTSDIR${PORT#.} && make fetch > /dev/null 2>&1 &
      START_SUBPROC=0;
    else
      echo "fetching: $CONCURRENTFETCH_NUM, subproc: $SUBPROC_NUM.  Waiting for sub processes to finish fetching"
      $SLEEP_LOC 1;
    fi
  done
done
wait
echo "Distifile fetch finished: `$DATE_LOC`"