r-wheeler

import spark.streaming.StreamingContext._
import spark.streaming.{Seconds, StreamingContext}
import spark.SparkContext._
import spark.storage.StorageLevel
import spark.streaming.examples.twitter.TwitterInputDStream
import com.twitter.algebird.HyperLogLog._
import com.twitter.algebird._

/**
 * Example of using HyperLogLog monoid from Twitter's Algebird together with Spark Streaming's

r-wheeler

As a follow up to my last post about no free lunch I wanted to show more state monad usage and recognize that you can change function return types in the middle of a for-comprehension. This allows each function to return different types of values from the computation along the way.

In the code below, you compose a function by using the for-comprehension. The yield part of the for-comprehension is a State object. When that state object is called, it returns a tuple of (state, value). You can use ._1 or ._2 to obtain the value of interest.

The last composed function shows an example of changing the state type (instead of changing the returned value type mentioned above) in the middle of the for-comprehension using scalaz IndexedState's iPut and iModify. This allows you to change the type of state as you go to adapt to different function's API needs.

import scalaz._
import scalaz.State._

r-wheeler

// Requires scalaz to be on the classpath

import scalaz._
import scalaz.State._

// The State monad was hard for me to understand because it
// seems as if you never actually store a value in the state and
// manipulate it inside of a method like you would in java.

// There are really two types of methods in the scalaz library

r-wheeler

import luigi
import luigi.scheduler
import luigi.worker
import logging as log
import socket
from datetime import datetime as dt
from ConfigParser import ConfigParser
from components import AssessSVMRegression
from components import CreateProteinList
from components import CreateReport

r-wheeler

import luigi
import time

class TimeTaskMixin(object):
    '''
    A mixin that when added to a luigi task, will print out
    the tasks execution time to standard out, when the task is
    finished
    '''
    @luigi.Task.event_handler(luigi.Event.PROCESSING_TIME)

r-wheeler

package object mail {

  implicit def stringToSeq(single: String): Seq[String] = Seq(single)
  implicit def liftToOption[T](t: T): Option[T] = Some(t)

  sealed abstract class MailType
  case object Plain extends MailType
  case object Rich extends MailType
  case object MultiPart extends MailType

r-wheeler

{-# LANGUAGE DataKinds         #-}
{-# LANGUAGE DeriveGeneric     #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE PolyKinds         #-}
{-# LANGUAGE TypeFamilies      #-}
{-# LANGUAGE TypeOperators     #-}

import           Control.Applicative
import           Control.Monad.IO.Class
import           Data.Aeson

r-wheeler

#author : Sujit Pal 

#Note: this is a python3 updated version of http://sujitpal.blogspot.fr/2014/12/semantic-similarity-for-short-sentences.html 
# by mathieu Chrétien (mchretien.pro@mail.com)

#contributor : Mathieu Chrétien

from __future__ import division
import nltk
from nltk.corpus import wordnet as wn

r-wheeler

class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
     """
     A version of :class:`concurrent.futures.ThreadPoolExecutor` that
     always uses native threads, even when threading is monkey-patched.

     .. versionadded:: 1.2a1
     """

     def __init__(self, max_workers):
         super(ThreadPoolExecutor, self).__init__(max_workers)

r-wheeler

class AttentionWithContext(Layer):
    """
        Attention operation, with a context/query vector, for temporal data.
        Supports Masking.
        Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]
        "Hierarchical Attention Networks for Document Classification"
        by using a context vector to assist the attention
        # Input shape
            3D tensor with shape: `(samples, steps, features)`.
        # Output shape