Antwnis

044a5b73d79fa4482277b7706c0f1b524edea0a4277115599c59bd375780a79c9a4dea40ed66aef20799aeee8cef8155029438a831a94815a8074cd1e3a7375856

Antwnis

version: '2'
services:
    zookeeper:
        image: "confluentinc/cp-zookeeper"
        container_name: zookeeper
        ports:
            - "2181:2181"
        environment:
            - ZOOKEEPER_CLIENT_PORT=2181
    kafka:

Antwnis

   for(Footer f : ParquetFileReader.readFooters(conf, fs, false)){
   for(BlockMetaData b : f.getParquetMetadata().getBlocks()){
         rowCount += b.getRowCount();
    }

Antwnis

package org.fannan.etl.examples

import cascading.pipe.joiner.{OuterJoin, LeftJoin}
import com.twitter.scalding._

class IUDJob(args: Args) extends Job(args) {

  val schema = List('CustID,'AccountID,'LastUpdateDate)
  
  val old_data = List(

Antwnis

## Maven ##
wget http://www.motorlogy.com/apache/maven/maven-3/3.2.1/binaries/apache-maven-3.2.1-bin.zip -O /usr/local/src/maven-3.2.1.zip
unzip /usr/local/src/maven-3.2.1.zip -d /opt
mv /opt/apache-maven-3.2.1 /opt/maven
ln -s /opt/maven/bin/mvn /usr/bin/mvn
bash -c "echo 'MAVEN_HOME=/opt/maven' > /etc/profile.d/maven.sh"
bash -c "echo 'MAVEN_OPTS=\"-Xmx2g -Xmx512m -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m\"' >> /etc/profile.d/maven.sh"
bash -c "echo 'export CLASSPATH=.' >> /etc/profile.d/maven.sh"
source /etc/profile.d/maven.sh

Antwnis

#!/bin/bash
# installs to /opt/gradle
# existing versions are not overwritten/deleted
# seamless upgrades/downgrades
# $GRADLE_HOME points to latest *installed* (not released)
gradle_version=2.2
mkdir /opt/gradle
wget -N http://services.gradle.org/distributions/gradle-${gradle_version}-all.zip
unzip -oq ./gradle-${gradle_version}-all.zip -d /opt/gradle
ln -sfnv gradle-${gradle_version} /opt/gradle/latest

Antwnis

export SCALA_VERSION=scala-2.11.5
sudo wget http://www.scala-lang.org/files/archive/${SCALA_VERSION}.tgz
sudo echo "SCALA_HOME=/usr/local/scala/scala-2.11.5" > /etc/profile.d/scala.sh
sudo echo 'export SCALA_HOME' >> /etc/profile.d/scala.sh
sudo mkdir -p /usr/local/scala
sudo -s cp $SCALA_VERSION.tgz /usr/local/scala/
cd /usr/local/scala/
sudo -s tar xvf $SCALA_VERSION.tgz
sudo rm -f $SCALA_VERSION.tgz
sudo chown -R root:root /usr/local/scala

Antwnis

sudo echo 'SBT_HOME=/usr/local/sbt/sbt-0.13.1' > /etc/profile.d/sbt.sh
sudo echo 'export SBT_HOME' >> /etc/profile.d/sbt.sh
sudo mkdir -p /usr/local/sbt
wget http://repo.scala-sbt.org/scalasbt/sbt-native-packages/org/scala-sbt/sbt/0.13.1/sbt.tgz
sudo -s cp sbt.tgz /usr/local/sbt/
cd /usr/local/sbt/
sudo -s tar xvf sbt.tgz
sudo mv sbt sbt-0.13.1
sudo rm -f sbt.tgz
sudo chown -R root:root /usr/local/sbt

Antwnis

import AssemblyKeys._

assemblySettings

net.virtualvoid.sbt.graph.Plugin.graphSettings

name := "hbasetest"

version := "1.0"

Antwnis

import cascading.pipe.Pipe
import cascading.tuple.{TupleEntry, Fields}
import com.twitter.scalding._

// One trait + one object = Custom Operations
trait HashOperations extends FieldConversions {
  def self: RichPipe
  def generateHash : Pipe = self
    .map(Fields.ALL -> 'hash) { te : TupleEntry =>
       val tuple = te.getTuple