milimetric

import signal
import re


class TimeoutError(Exception):
    pass


class timeout:

milimetric

/* Common setup, two variants follow
 */
use test;

    set tablename=webrequest_esams0;

    add jar /home/otto/hive-serdes-1.0-SNAPSHOT.jar;
    add jar /usr/lib/hive/lib/hive-contrib-0.10.0-cdh4.3.1.jar;
 create temporary function rowSequence AS 'org.apache.hadoop.hive.contrib.udf.UDFRowSequence';

milimetric

projectview_ready = HiveTriggeredHQLTaskFactory(
    'run_hql_and_arhive',
    default_args=default_args,
    ...
)

archive = ArchiveTaskFactory(...)

projectview_ready.sensors() >> projectview_ready.etl() >> archive()

milimetric

#!/bin/bash
#
# This script does the following:
#       0. reads four arguments from the CLI, in order, as YEAR, MONTH, DAY, HOUR
#       1. downloads the specified hour worth of data from http://dumps.wikimedia.org/other/pagecounts-raw/
#       2. extracts the data into hdfs
#       3. creates a partition on a hive table pointing to this data
#

print_help() {

milimetric

use wmf;

-- new data
 select coalesce(c.country, g.country_code) as country,
        sum(edit_count) as edits,
        sum(namespace_zero_edit_count) as namespace_zero_edits

   from geoeditors_edits_monthly g
            inner join
        (select distinct dbname

milimetric

 select ar_id, ar_namespace, ar_title, NULL as ar_text, NULL as ar_comment, NULL as ar_comment_id,
        case when ar_deleted&4 != 0 then null when ar_actor = 0
        then ar_user else COALESCE( actor_user, 0 ) END AS ar_user,
        case when ar_deleted&4 != 0 then null when ar_actor = 0
        then ar_user_text else actor_name END AS ar_user_text,
        if(ar_deleted&4 <> 0,0,ar_actor) as ar_actor, ar_timestamp, ar_minor_edit, NULL as ar_flags, ar_rev_id,
        case when ar_deleted&1 != 0 then null when content_id is NULL then ar_text_id
        else content_id end as ar_text_id,
        ar_deleted, if(ar_deleted&1 <> 0,null,ar_len) as ar_len,

milimetric

with users_with_revisions as (
 select event_user_id,
        event_timestamp
   from mediawiki_history
  where event_entity = 'revision'
    and event_type = 'create'
    and snapshot = '2018-05'
    and wiki_db = 'enwiki'
)

milimetric

<template>
<!-- In Vue, $el is this root element defined in the template section -->
<span></span>
</template>

<script>
// the script-loader webpack plugin has to be used to hack the oojs-ui files directly into script tags
// because they have no modularization whatsoever (AMD, ES6, etc.)
import 'script-loader!oojs/dist/oojs.jquery'
import 'oojs-ui/dist/oojs-ui-core'

milimetric

{
   "dataSources" : [
      {
         "spec" : {
            "dataSchema" : {
               "dataSource" : "pageviews-hourly",
               "metricsSpec" : [
                   {
                     "name" : "view_count",
                     "type" : "longSum",

milimetric

# NOTE: required for the following to work:
# !pip    install    pymysql\n",
# !git    clone    https://gerrit.wikimedia.org/r/p/operations/mediawiki-config\n",
# !cd    mediawiki-config    &&    git    pull    origin    master"
import    pymysql
import    ipaddress
import    os

connection = pymysql.connect(
    host='analytics-store.eqiad.wmnet',