ottomata

import urllib2
import ast
from pprint import pprint as pp
sites = ['eqiad', 'esams', 'ulsfo']
roles = ['bits', 'mobile', 'text', 'upload']
base_url = 'http://noc.wikimedia.org/pybal'

config = []
for site in sites:
    for role in roles:

ottomata

root@analytics1021:~# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/md3         28G  8.1G   19G  31% /
udev             24G  4.0K   24G   1% /dev
tmpfs           9.5G  412K  9.5G   1% /run
none            5.0M     0  5.0M   0% /run/lock
none             24G     0   24G   0% /run/shm
/dev/sdl1       1.8T  427G  1.4T  24% /var/spool/kafka/l
/dev/sdc1       1.8T  417G  1.4T  23% /var/spool/kafka/c
/dev/sdg1       1.8T  417G  1.4T  23% /var/spool/kafka/g

ottomata

2014-03-04 18:38:46.668346 [search_9200] Merged enabled server elastic1007.eqiad.wmnet, weight 30
2014-03-04 18:38:46.670842 [search_9200] New enabled server elastic1013.eqiad.wmnet, weight 30
2014-03-04 18:38:46.672395 [search_9200] New enabled server elastic1014.eqiad.wmnet, weight 30
2014-03-04 18:38:46.674016 [search_9200] New enabled server elastic1015.eqiad.wmnet, weight 30
2014-03-04 18:38:46.675528 [search_9200] New enabled server elastic1016.eqiad.wmnet, weight 30
2014-03-04 18:38:46.676706 Resolved elastic1013.eqiad.wmnet to addresses 10.64.48.10
2014-03-04 18:38:46.677902 Resolved elastic1014.eqiad.wmnet to addresses 10.64.48.11
2014-03-04 18:38:46.679163 Resolved elastic1015.eqiad.wmnet to addresses 10.64.48.12
2014-03-04 18:38:46.680241 Resolved elastic1016.eqiad.wmnet to addresses 10.64.48.13
2014-03-04 18:38:46.704062 [search_9200 IdleConnection] elastic1013.eqiad.wmnet (enabled/up/pooled): Connection established.

ottomata

node 'limn1.eqiad.wmflabs' {
    include webserver::apache

    # make sure /var/log/apache2 is readable by wikidevs for debugging.
    # This won't make the actual log files readable, only the directory.
    # Individual log files can be created and made readable by
    # classes that manage individual sites.
    file { '/var/log/apache2':
        ensure  => 'directory',
        owner   => 'root',

ottomata

#!/usr/bin/python

import datetime, os, csv, sys
listen_port = '9199'

data = {}
with sys.stdin as csvfile:
    for row in csv.reader(csvfile):
        date = datetime.datetime.strptime(row[0], "%Y%m%d%H%M%S").strftime('%Y-%m-%d %H:%M')
        if date not in data:

ottomata

root@analytics1010:~# sudo -u hdfs hdfs dfsadmin -printTopology
Rack: /eqiad/A/2
   10.64.5.11:50010 (analytics1011.eqiad.wmnet)
   10.64.5.12:50010 (analytics1012.eqiad.wmnet)
   10.64.5.13:50010 (analytics1013.eqiad.wmnet)

Rack: /eqiad/C/7
   10.64.36.114:50010 (analytics1014.eqiad.wmnet)
   10.64.36.115:50010 (analytics1015.eqiad.wmnet)
   10.64.36.116:50010 (analytics1016.eqiad.wmnet)

ottomata

#######################################################################
#                                                                     #
#                    kafkatee configuration file                      #
#                                                                     #
#                                                                     #
#######################################################################
#                                                                     #
# Syntax:                                                             #
#  <property-name> = <value>                                          #
#  input <type args..>                                                #

ottomata

cd  /a/squid/archive/glam_nara;
for f in glam_nara*.gz; do
    echo $f;
    year=${f:18:4};
    month=${f:22:2};
    day=${f:24:2};
    hdfs_dir=/user/otto/data/external/webrequest_glam_nara/daily/$year/$month/$day;
    echo hdfs dfs -mkdir $hdfs_dir '&&' zcat $f '|' /home/otto/separate_udp_filter_geocoded_field.sh '|' hdfs dfs -put - $hdfs_dir/glam_nara.tsv.log-$year$month$day;
    hdfs dfs -mkdir $hdfs_dir && zcat $f | /home/otto/separate_udp_filter_geocoded_field.sh | hdfs dfs -put - $hdfs_dir/glam_nara.tsv.log-$year$month$day;
done

ottomata

In [21]: command
Out[21]: ['/bin/echo', 'hi', '|', '/usr/bin/env', 'sed', '-e', "'s@hi@wee@'"]

In [22]: command_string
Out[22]: "/bin/echo hi | /usr/bin/env sed -e 's@hi@wee@'"

In [23]: p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)

In [24]: stdout, stderr = p.communicate()

ottomata

#######################################################################
#                                                                     #
#                varnishkafka configuration file                      #
#                                                                     #
#                                                                     #
#######################################################################
#                                                                     #
# Syntax:                                                             #
# <property-name> = <value>                                           #
#                                                                     #