count_seqs.sh

# udp2log:
# separate hosts
cd /a/otto/mobile-sleuth.2/udp2log.orig
for host in cp104{1,2,3,4}; do
    grep -P "^$host" mobile.2013-01-13_11.45.00-13.15.00.log | awk '{print $2 " " $3}' | sort -n > ../$host.udp2log.seq
done


# hdfs:
# separate hosts
cd /a/otto/mobile-sleuth.2/hdfs.orig
for host in cp104{1,2,3,4}; do
    grep -P "^\d+\s+$host" mobile.hdfs.2013-01-23_12.00.00-13.15.00.log | awk '{print $3 " " $4}' | sort -n > ../$host.hdfs.seq
done

cd /a/otto/mobile-sleuth.2/
for host in cp104{1,2,3,4}; do
    first_seq=$(head -n 1 $host.udp2log.seq | awk '{print $1}') # && echo $first_seq
    last_seq=$(tail -n 1 $host.udp2log.seq | awk '{print $1}') #  && echo $last_seq

    should_be=$(echo "$last_seq - $first_seq + 1" | bc) #&& echo $should_be

    # count lines between $first_seq and $last_seq
    hdfs_count=$(sed -n -e "/^$first_seq/,/^$last_seq/p" $host.hdfs.seq | wc -l) #&& echo $hdfs_count
    udp2log_count=$(sed -n -e "/^$first_seq/,/^$last_seq/p" $host.udp2log.seq | wc -l) #&& echo $udp2log_count

    echo "----"
    echo "$host:"
    echo "  should_be:     $should_be"; echo "  udp2log_count: $udp2log_count"; echo "  hdfs_count:    $hdfs_count";
done

gaps_per_second.sh

awk 'NR-1{if($1!=(_+1))print $2}{_=$1}' cp1044.udp2log.seq | uniq -c

gistfile1.txt

----
cp1041:
  should_be:     2183234
  udp2log_count: 2073716
  hdfs_count:    2073716
----
cp1042:
  should_be:     2185238
  udp2log_count: 2075082
  hdfs_count:    2075081
----
cp1043:
  should_be:     2197387
  udp2log_count: 2086935
  hdfs_count:    2086935
----
cp1044:
  should_be:     2197839
  udp2log_count: 2086864
  hdfs_count:    2086864

tcpdump

# to capture udp2log ASCII data from tcpdump:
sudo tcpdump udp port 8420 -A -w - | strings

For readability:

totals:
    should_be:   8,763,698
    udp2log:     8,322,597 (-5.033%, -441,101)
    hdfs:        8,322,596 (-5.033%, -441,102)

cp1041:
    should_be:   2,183,234
    udp2log:     2,073,716 (-5.016%, -109,518)
    hdfs:        2,073,716 (-5.016%, -109,518)
cp1042:
    should_be:   2,185,238
    udp2log:     2,075,082 (-5.041%, -110,156)
    hdfs:        2,075,081 (-5.041%, -110,157)
cp1043:
    should_be:   2,197,387
    udp2log:     2,086,935 (-5.027%, -110,452)
    hdfs:        2,086,935 (-5.027%, -110,452)
cp1044:
    should_be:   2,197,839
    udp2log:     2,086,864 (-5.049%, -110,975)
    hdfs:        2,086,864 (-5.049%, -110,975)