ottomata

# Usage:
# eventlogging_processor_bench.sh <num_processors> <time_to_run> <consumer_group>
#

eventlogging_path=/home/otto/EventLogging/server
export PYTHONPATH=$eventlogging_path


num_processors=${1:-1}
time_to_run=${2:-10}

ottomata

Ryan_Lane		3:11    ottomata: yo
ottomata		3:12    hiyaaaa
Ryan_Lane		3:12    so, yeah, let's delete those two gerrit repos
ottomata		3:12    oh ok!
Ryan_Lane		3:12    git-deploy and sartoris
ottomata		3:12    awesome
ottomata		3:12    our git-deploy is the python port of the perl git-deploy, and sartoris is the salt backend
Ryan_Lane		3:12    it'll help reduce the confusion factor
Ryan_Lane		3:12    nop
ottomata		3:12    and we want to rename them both to trebuchet stuff

ottomata

### udp2log packet loss monitoring
pipe 10 /usr/bin/packet-loss 10 '\t' >> /var/log/udp2log/packet-loss.log

### Capture all logs with 'zero=' set.  The X-Analytics header is set with this
# by mobile varnish frontends upon getting a Wikipedia Zero request.
pipe 1 /bin/grep -P 'zero=\d{3}-\d{2}' >> /a/log/webrequest/zero.tsv.log

### All edits
pipe 1 /usr/bin/udp-filter -F '\t' -p action=submit,action=edit >> /a/log/webrequest/edits.tsv.log

ottomata

root@neon:~# check_ganglia -q -g nickel.wikimedia.org -p 8654 -H cp3019.esams.wikimedia.org --list | grep -P 'queue|cnt|rtt' | grep -v analytics1021 | sort
kafka.rdkafka.brokers.analytics1022-eqiad-wmnet_9092.22.outbuf_cnt (uint32, ) 22144
kafka.rdkafka.brokers.analytics1022-eqiad-wmnet_9092.22.rtt.avg (uint32, microseconds) 3042759
kafka.rdkafka.brokers.analytics1022-eqiad-wmnet_9092.22.rtt.cnt (uint32, ) 47
kafka.rdkafka.brokers.analytics1022-eqiad-wmnet_9092.22.rtt.max (uint32, microseconds) 3718441
kafka.rdkafka.brokers.analytics1022-eqiad-wmnet_9092.22.rtt.min (uint32, microseconds) 2266346
kafka.rdkafka.brokers.analytics1022-eqiad-wmnet_9092.22.waitresp_cnt (uint32, ) 11
kafka.rdkafka.topics.webrequest_bits.partitions.0.msgq_cnt (uint32, ) 0
kafka.rdkafka.topics.webrequest_bits.partitions.0.xmit_msgq_cnt (uint32, ) 0
kafka.rdkafka.topics.webrequest_bits.partitions.1.msgq_cnt (uint32, ) 0

ottomata

def _update_artifacts(location):
    artifact_config = '{0}/artifacts.json'.format(location)

    if os.path.exists(artifact_config):
        with open(artifact_config) as json_file:
            artifacts = json.load(json_file)

        for artifact in artifacts:
            if artifact['path'].startswith('/'):

ottomata

# using this because OS X doesn't have readlink -f :/
alias realpath='python -c "import os,sys; print os.path.realpath(sys.argv[1])" '

artifact_repository_path=/path/to/archiva/repo
git_fat_store_path=/path/to/git-fat-store

find $artifact_repository_path -type f -name "*.jar" | xargs shasum | while read line; do
  sum=$(echo $line | awk '{print $1}'); file=$(echo $line | awk '{print $2}');
  ln -svf $(realpath $file) $git_fat_store_path/$sum;
done

ottomata

com.atlassian.xmlrpc:atlassian-xmlrpc-binder-annotations:jar:0.9:compile
com.atlassian.xmlrpc:atlassian-xmlrpc-binder-server-spring:jar:0.9:runtime
com.atlassian.xmlrpc:atlassian-xmlrpc-binder-server:jar:0.9:compile
com.atlassian.xmlrpc:atlassian-xmlrpc-binder-server:jar:0.9:runtime
com.atlassian.xmlrpc:atlassian-xmlrpc-binder-support:jar:0.9:compile
com.atlassian.xmlrpc:atlassian-xmlrpc-binder-support:jar:0.9:runtime
com.atlassian.xmlrpc:atlassian-xmlrpc-binder:jar:0.9:compile

com.google.code.cli-parser:cli:jar:7:compile

ottomata

# producer thread for analytics1021
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 94.64    0.141256         457       309           poll
  5.36    0.008001        8001         1           restart_syscall
  0.00    0.000000           0        32           madvise
  0.00    0.000000           0       285           sendmsg
  0.00    0.000000           0       773       257 recvmsg
  0.00    0.000000           0        11           futex
------ ----------- ----------- --------- --------- ----------------

ottomata

root@cp3019:~# tcptrace -lr an22.tcpdump.produce.dump
1 arg remaining, starting with 'an22.tcpdump.produce.dump'
Ostermann's tcptrace -- version 6.6.7 -- Thu Nov  4, 2004

17291 packets seen, 17291 TCP packets traced
elapsed wallclock time: 0:00:00.170699, 101295 pkts/sec analyzed
trace file elapsed time: 0:04:00.459721
TCP connection info:
1 TCP connection traced:
TCP connection 1:

ottomata

[2014-02-21 05:36:33,613] 119762261 [isr-expiration-thread-1] INFO  kafka.cluster.Partition  - Partition [webrequest_bits,2] on broker 21: Shrinking ISR for partition [webrequest_bits,2] from 21,22 to 21
[2014-02-21 05:36:33,932] 119762580 [main-SendThread(analytics1024.eqiad.wmnet:2181)] INFO  org.apache.zookeeper.ClientCnxn  - Client session timed out, have not heard from server in 13515ms for sessionid 0x425d903a351257, closing socket connection and attempting reconnect
[2014-02-21 05:36:34,057] 119762705 [kafka-logcleaner-1] INFO  kafka.log.Log  - [Kafka Log on Broker 21], Deleting log segment 2359785906 from webrequest_mobile-6
[2014-02-21 05:36:34,070] 119762718 [main-EventThread] INFO  org.I0Itec.zkclient.ZkClient  - zookeeper state changed (Disconnected)
[2014-02-21 05:36:34,593] 119763241 [kafka-logcleaner-1] INFO  kafka.log.OffsetIndex  - Deleting index /var/spool/kafka/d/data/webrequest_mobile-6/00000000002359785906.index
[2014-02-21 05:36:34,594] 119763242 [kafka-logcleaner-1] INFO  kafka.log.Log