rjurney

import javax.ws.rs.{QueryParam, GET, Produces, Path}
import scala.Array
import javax.ws.rs.core.{Response, MediaType}
import org.slf4j.{LoggerFactory, Logger}
import org.joda.time.format.{ISODateTimeFormat, DateTimeFormatter}
import org.joda.time.DateTimeZone

import com.securityx.modelfeature.dao.BeaconsDao
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule

rjurney

Russells-MacBook-Pro:spark rjurney$ git checkout -b pyspark-inputformats mlnick/pyspark-inputformats
fatal: Cannot update paths and switch to branch 'pyspark-inputformats' at the same time.
Did you intend to checkout 'mlnick/pyspark-inputformats' which can not be resolved as commit?

rjurney

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import com.salesforce.phoenix.jdbc.PhoenixDriver

Java::com.salesforce.phoenix.jdbc.PhoenixDriver
DriverManager.registerDriver("com.salesforce.phoenix.jdbc.PhoenixDriver")
conn = DriverManager.get_connection("jdbc:phoenix:hiveapp1,hiveapp2:2181")

log4j:WARN No appenders could be found for logger (org.apache.hadoop.conf.Configuration).

rjurney

[hivedata@hivedev1 lipstick-server]$ ./grailsw run-app
| Running Grails application
| Error 2014-06-26 15:14:08,239 [localhost-startStop-1] ERROR context.GrailsContextLoader  - Error initializing the application: Error creating bean with name 'transactionManagerPostProcessor': Initialization of bean failed; nested exception is org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'transactionManager': Cannot resolve reference to bean 'sessionFactory' while setting bean property 'sessionFactory'; nested exception is org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'sessionFactory': Cannot resolve reference to bean 'hibernateProperties' while setting bean property 'hibernateProperties'; nested exception is org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'hibernateProperties': Cannot resolve reference to bean 'dialectDetector' while setting bean property 'properties' with key [hibernate.dialect]; nested 

rjurney

from pig_util import outputSchema
import sys, os, re

@outputSchema('matches:bag{t:tuple(name:chararray)}') # I am a pig schema
def match_names(one_name, all_names): #all_names is an array with 150,000 string elements
  match_pairs = []
  for name_tuple in all_names:
    name = name_tuple[0]
    match = one_name.find(name)
    if match >= 0:

rjurney

grouped = GROUP links BY (sourceNameOrIp, destinationNameOrIp, $filter);
sorted_grouped = FOREACH grouped {
    sorted_datetimes = ORDER links BY date_time;
    GENERATE FLATTEN(group) AS (sourceNameOrIp, destinationNameOrIp, $filter), 
             sorted_datetimes.(date_time) AS sorted_datetimes, 
             sorted_datetimes.(log_hash) as sorted_loghashes;
};
sorted_grouped = FOREACH sorted_grouped GENERATE sourceNameOrIp, destinationNameOrIp, $filter, sorted_datetimes, sorted_loghashes;
has_beacons = FILTER sorted_grouped by SIZE(sorted_datetimes) > 1;
has_beacons = FOREACH has_beacons GENERATE SIZE(sorted_datetimes) AS total;

rjurney

{
count: 63,
group: {
type: "cefSignatureId",
groupField: "206",
periodSeconds: 600
},
timeSeries: [
{
bitsOutPerSecond: 338021.6,

rjurney

{
    count: 63,
    group: {
        type: "cefSignatureId",
        groupField: "206",
        periodSeconds: 600
    },
    timeSeries: [
        {
            bitsOutPerSecond: 338021.6,

rjurney

0: jdbc:phoenix:hiveapp1> select * from DEV_ENTROPY_TIME_SERIES limit 10;
+------------+-------------+----------------+------------+----------+
|    TYPE    | GROUP_FIELD | PERIOD_SECONDS | DATE_TIME  | ENTROPY  |
+------------+-------------+----------------+------------+----------+
| cefSignatureId | 0           | 3600           | 2014-06-11T16:00:00.000Z | 0.545344 |
| cefSignatureId | 0           | 3600           | 2014-06-11T16:10:00.000Z | 0.204304 |
| cefSignatureId | 0           | 3600           | 2014-06-11T16:20:00.000Z | 0.504859 |
| cefSignatureId | 0           | 3600           | 2014-06-11T16:30:00.000Z | 0.764432 |
| cefSignatureId | 0           | 3600           | 2014-06-11T16:40:00.000Z | 0.628618 |
| cefSignatureId | 0           | 3600           | 2014-06-11T16:50:00.000Z | 0.681011 |

rjurney

{
    u'applicationProtocol': u'http',
    u'bytesIn': 1095,
    u'bytesOut': 10733,
    u'cefSignatureId': u'200',
    u'destinationAddress': None,
    u'destinationDnsDomain': u'on24.com',
    u'destinationHostName': u'webcast.on24.com',
    u'destinationMacAddress': None,
    u'destinationNameOrIp': u'webcast.on24.com',