kerinin

set default_parallel 92;

read_rate_records = LOAD 's3://oib-mapreduce/output/folder_scan_rollups/2013/06/2013-02-15' AS (
        scanned_at:chararray, 
        received_at:chararray, 
        mm_id,
        source:chararray, 
        folder:chararray, 
        read, 
        subject:chararray,

kerinin

  def get_message_infos_after_date(ymail_folder, last_date)
    messages        = []
    oldest_found    = Time.now.to_i
    last_date       = last_date.to_i
    offset          = 0
    while oldest_found > last_date
      hash            = self.request('ListMessages', :fid => ymail_folder.fid, :sortKey => "date", :sortOrder => "down",
                                   :startInfo => offset, :startMid => offset, :numInfo => 100, :numMid => 100)
      found_messages  = hash['result']['messageInfo'].map{|h| Ymail::YmailApi::DataTypes::MessageInfo.make_from_hash(h,ymail_folder)}
      messages.push(*found_messages)

kerinin

class DevelopersController < ApplicationController
  respond_to :hal

  FILTERS = {
    :created_since        => ->(p,d) {d.where(:created_at.gte => p.to_i)},
    :last_activity_since  => ->(p,d) {d.with_last_activity_after(p)},
    :email_address        => ->(p,d) {d.where(email_address: p)},
    :api_key              => ->(p,d) {d.where(api_key: p)}
  }
  

kerinin

class ResponseObject
  def initialize
    @handlers = Hash.new { [] }
    yield self
  end
  
  def do(state, *args)
    @state = state
    @args = args
  end

kerinin

      def subnet_ids
        # TODO: Changed `availability_zones` key to `subnets` for backwards compatibility. 
        #   Need to change it back here and in the config after all components and stacks 
        #   are ready.
        availability_zones.map do |az|
          subnet_id(az)
        end.compact
      end

kerinin

# Basics - if you define an enumerable, you can evaluate it lazily
class Foo
  include Enumerable
  def each(&block)
    (1..20).to_a.each do |i|
      yield i
      puts "yielded #{i}"
    end
  end
end

kerinin

# This is for building a JSON request to the monitoring service
MonitorClient.group(name: 'my group') do |g|
  g.add_watch(name: 'Something fragile') do |w|
    w.description = 'information about watch'
    w.docs = 'http://triage.com'
    w.owners = ['ryan@example.com']
    w.metric = MonitorClient::Metric::Graphite.new(target: 'foo.bar.baz')
    w.policy = never_be_zero
    
    w.add_action MonitorClient::Action::Email.new

kerinin

Cluster Management

Problem Statement

Distributed computing environments face two significant problems

• Computing resources need to be assigned to computing tasks in a coordinated way (ie, if I have two computers and two tasks, I don't want both of the computers trying to do the same task)
• Process failure needs to be detected and responded to (ie if I run out of disk space, another process should take over my task)

An ideal solution would satisfy a couple objectives

kerinin

-- This will need to be built and put on the machine you're running Hive from
add jar json-serde-1.1.9.3-SNAPSHOT-jar-with-dependencies.jar;

-- Creates the table based on the JSON schema
CREATE TABLE json_nested_test (
  metadata struct<uuid:string,md5_email:string,received_at:string,message_id:string,from_domain:string,message_signature:string,provider:string>,
  headers map<string,struct<length:string,match_counts:map<string,string>,text_hash_counts:array<array<string>>,address_hashes:array<string>>>,
  parts array<struct<mime_type:string,length:string,stripped_length:string,match_counts:map<string,string>,urls:array<struct<domain:string,sha:string>>,images:array<struct<domain:string,sha:string,pixel_count:string>>,text_hash_counts:array<array<string>>>>,
  attachments array<struct<mime_type:string,bytesize:string,filename_sha:string,filetype:string>>
)

kerinin

gem 'faraday'
gem 'faraday_middleware'