test 'apt-get update' for a given "mode" release and mirror written to get reasonable information for bug 1177432 (http://padl.lv/1177432)

README.md

Summary

These tools are used to show collect information on performance and cost of a 'apt-get update'.

You need to have chdist from devscripts package to use it.

The results shown here are on a cloud with a local mirror, so latency is low and bandwidth high.

Usage

Collect some data for runs of both 'cloud' and 'cd' modes. This will call test-archive-update multiple times and copy out the output to out.d.

$ ./collect-data out.d

Then summarize show some information on what all happened. This shows the HTTP status code for each GET that apt did and how much data was downloaded. The output format is the stderr output of apt-update -q -o Debug::Acquire::http=1.

$ ./summarize-http out.d/cloud/clean/update.err
...
data bytes: 14431826
total GETS: 27
200: 16
404: 11

$ ./summarize-http out.d/cloud/noclean/update.err
...
data bytes: 0
total GETS: 27
304: 16
404: 11

$ ./summarize-http out.d/cd/clean/update.err
...
data bytes: 14893753
total GETS: 58
200: 37
404: 21

$ ./summarize-http out.d/cd/noclean/update.err
...
data bytes: 0
total GETS: 58
304: 37
404: 21

Comments

In the examples above 'cloud' represents the old cloud apt sources.list, which did not contain restricted or multiverse or backports. The big thing to notice is that every apt-get update ends up doing the same amount of GETS. Many of them are 'If-Modified-Since' requests and an unmodified response (304).

Also to note as significant is the number of 404. These http stat operations are also expensive.

On Jan 19, 2016 on xenial (1.1.10) I noticed that 'apt-get update' was only getting InRelease files and seemed to not be doing an obscene amount of GETs either. See below for improvements:

$ ./collect-data out.d xenial
$ for e in ./out.d/*/*/update.err; do echo == $e ==; ./summarize-http $e; done > out; grep -v '^[^[]' out
    == ./out.d/cd/clean/update.err ==
data bytes: 7595517
total GETS: 7
200: 7
== ./out.d/cd/noclean/update.err ==
data bytes: 0
total GETS: 4
304: 4
== ./out.d/cloud/clean/update.err ==
data bytes: 7522001
total GETS: 6
200: 6
== ./out.d/cloud/noclean/update.err ==
data bytes: 0
total GETS: 3
304: 3

collect-data

#!/bin/sh
mirror=${MIRROR:-http://nova.clouds.archive.ubuntu.com/ubuntu}
outd="${1:-out.d}"
rel=${2:-trusty}
rm -Rf "$outd"
for mode in cloud cd; do
  for clean in clean noclean; do
     sdir="$outd/$mode/$clean"
     echo === $sdir ===
     mkdir -p "$sdir"
     [ "$clean" = "clean" ] && cflag="--clean" || cflag=""
     echo ./test-archive-update "--mirror=$mirror" $cflag $mode $rel &&
     ./test-archive-update "--mirror=$mirror" $cflag $mode $rel &&
        cp -a ~/.chdist/$rel/var "$sdir/" &&
        cp $rel-$mode-$clean.out $sdir/update.out && 
        cp $rel-$mode-$clean.err $sdir/update.err ||
        exit
  done
done

summarize-http

#!/bin/bash
[ "$1" = "-h" -o "$1" = "--help" ] && { echo "${0} log"; exit 0; }
input=""
[ -n "$1" -a "$1" != "-" ] && input="$1"
if [ -n "$input" ]; then
   [ -f "$input" ] || { echo "$input: not a file"; exit 1; }
   exec <"$input"
fi

set -f

fail() { echo "$@" 1>&2; exit 1; }
declare -A STATUS_COUNTS
GETS=0
BYTES_DATA=0
result() {
    local status="$1" path="$2" bytes="$3"
    fmt="[%s] %7d %s"
    printf "$fmt\n" "$status" "$bytes" "$path"
    STATUS_COUNTS[$status]=$((${STATUS_COUNTS[$status]}+1))
    GETS=$(($GETS+1))
    case "$status" in
        200)
          BYTES_DATA=$(($BYTES_DATA+$bytes));;
    esac
}

parsedata() {
   local path="" finished="false" state=none bytes="" path="" status=""
   local cstate=none
   while :; do
      read line || break
      if [ -z "$line" ]; then
          case "$cstate" in
              get) cstate="trans";;
              response)
                  result "$status" "$path" "$bytes"
                  cstate="none";;
              none) :;;
          esac
          continue
      fi
      #echo "$cstate.$1"
      set -- $line
      case "$cstate.$1" in
          none.GET)
              status=""; path=""; bytes="";
              path="$2";
              cstate="get";;
          trans.HTTP*/*)
              cstate="response";
              status=$2;;
          response.Content-Length:) bytes="$2";;
          none.GET|response.HTTP*)
              fail "bad state / $1: cstate=$cstate 1=$1";;
      esac
   done
   [ "$cstate" = "none" ] || result "$status" "$path" "$bytes"
   echo "data bytes: $BYTES_DATA"
   echo "total GETS: $GETS"
   for status in ${!STATUS_COUNTS[@]}; do
      echo "${status}:" "${STATUS_COUNTS[$status]}"
   done
}

sed -e 's/\r//g' | parsedata

test-archive-update

#!/bin/bash

VERBOSITY=0
TEMP_D=""

error() { echo "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }

Usage() {
    cat <<EOF
Usage: ${0##*/} [ options ] mode [release]

   run apt-get for a given release/mode

   options:
      -p | --proxy  P    use http proxy
      -m | --mirror M    use mirror M
      -c | --clean       clean
      -S | --no-source   do not add deb-src

EOF
}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
debug() {
    local level=${1}; shift;
    [ "${level}" -gt "${VERBOSITY}" ] && return
    error "${@}"
}

gen_list() {
  local ltype=$1 rel=$2 uri=$3 src=${4}
  
  local debs="deb" dists comps
  if [ ${src:-0} != "0" ]; then
     debs="deb deb-src"
  fi
  local deb dist comp
  case "$ltype" in
    cloud)
      dists="_ -updates -security"
      comps="main universe"
      ;;
    cd)
      dists="_ -updates -security -backports"
      comps="main restricted universe multiverse"
      ;;
    *) fail "bad type $ltype: expected 'cloud', 'cd'";;
  esac
  for deb in $debs; do
    for dist in $dists; do
      [ "$dist" = "_" ] && dist=""
      for comp in $comps; do
        echo "$deb" "$uri" "$rel$dist" "$comp"
      done
    done
  done
}

main() {
    local short_opts="chm:p:Sv"
    local long_opts="clean,help,mirror:,no-source,proxy:,verbose"
    local getopt_out=""
    local getopt_out=$(getopt --name "${0##*/}" \
        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
        eval set -- "${getopt_out}" ||
        bad_Usage

    local mirror="http://archive.ubuntu.com/ubuntu/" proxy="" clean=false
    local mode="" rel="" src=1

    while [ $# -ne 0 ]; do
        cur=$1; next=$2;
        case "$cur" in
            -c|--clean) clean=true;;
            -h|--help) Usage ; return 0;;
            -m|--mirror) mirror=$next; shift;;
            -p|--proxy) proxy=$next; shift;;
            -S|--no-source) src=0;;
            -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
            --) shift; break;;
        esac
        shift;
    done

    [ $# -eq 2 -o $# -eq 1 ] ||
        bad_Usage "expected 1 or 2 args. got: $*"
    mode="$1"
    if [ $# -eq 2 ]; then
        rel="$2"
    else
        rel=$(lsb_release -sc) ||
            { error "failed to get rel from lsb_release -sc"; return 1; }
    fi

    local ddir="$HOME/.chdist/$rel"

    if $clean || [ ! -d "$ddir" ]; then
        rm -Rf "$ddir" ||
            { error "failed to clean $ddir"; return 1; }
      $clean || error "warning: clean not specified, but creating $rel"
      chdist create "$rel" "$mirror" ||
        { error "failed: chdist create $rel $mirror"; return 1; }
    fi

    local apt_d="$ddir/etc/apt"

    debug 1 "writing $apt_d/sources.list"
    gen_list "$mode" "$rel" "$mirror" > "$apt_d/sources.list" ||
        { error "failed gen_list $mode $rel $mirror"; return 1; }

    local n=clean
    $clean || n=noclean
    time env ${proxy:+"http_proxy=${proxy}"} \
        chdist apt-get "$rel" update -q -o Debug::Acquire::http=1 > $rel-$mode-$n.out 2>$rel-$mode-$n.err
}

main "$@"

# vi: ts=4 expandtab