earthquakesan

Keybase proof

I hereby claim:

• I am earthquakesan on github.
• I am earthquakesan (https://keybase.io/earthquakesan) on keybase.
• I have a public key whose fingerprint is 0CF0 6C02 DF41 A392 79DE 72A6 BD69 BF33 9056 721B

To claim this, I am signing this object:

earthquakesan

#!/bin/bash

# Setup
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF
DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
CODENAME=$(lsb_release -cs)

# Add the repository
echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | \
  sudo tee /etc/apt/sources.list.d/mesosphere.list

earthquakesan

LANGUAGE=en_US.UTF-8
LC_ALL=en_US.UTF-8
LC_PAPER=en_US.UTF-8
LC_ADDRESS=en_US.UTF-8
LC_MONETARY=en_US.UTF-8
LC_NUMERIC=en_US.UTF-8
LC_TELEPHONE=en_US.UTF-8
LC_IDENTIFICATION=en_US.UTF-8
LC_MEASUREMENT=en_US.UTF-8
LC_CTYPE=en_US.UTF-8

earthquakesan

# -*- coding: utf-8 -*-
import re
import distutils.sysconfig as sysconfig
import os

__doc__ = '''set-me-up.

Usage:
    set-me-up <projectdir>
'''

earthquakesan

build:
	build/mvn -DskipTests clean package
up:
	docker network create spark-net
	docker-compose -f docker-compose.yml up -d
down:
	docker-compose -f docker-compose.yml down
	docker-compose -f docker-compose-spark-app.yml down
	docker network rm spark-net
app:

earthquakesan

FROM bde2020/spark-submit:2.1.0-hadoop2.8-hive-java8

RUN mkdir -p /app
COPY examples/target/original-spark-examples_2.11-2.3.0-SNAPSHOT.jar /app/spark-examples.jar

ENV SPARK_MASTER_NAME spark-master
ENV SPARK_MASTER_PORT 7077
ENV SPARK_APPLICATION_JAR_LOCATION /app/spark-examples.jar
ENV SPARK_APPLICATION_MAIN_CLASS org.apache.spark.examples.SparkPi
ENV SPARK_APPLICATION_ARGS ""

earthquakesan

version: "2.1"
services:
  app:
    build: .
    networks:
      - spark-net
networks:
  spark-net:
    external:
      name: spark-net

earthquakesan

## JVM configuration

################################################################
## IMPORTANT: JVM heap size
################################################################
##
## You should always set the min and max JVM heap
## size to the same value. For example, to set
## the heap to 4 GB, set:
##

earthquakesan

import org.apache.spark.{SparkConf, SparkContext}

object ICTCSApacheLogParsing {
  def main(args: Array[String]): Unit = {
    val config = new SparkConf().setMaster("local[4]").setAppName("My Application")
    val sc = new SparkContext(config)

    val apacheLog = sc.textFile("/path/to/access_log")

    case class ApacheLog(ipAddress: String, date: String, statusCode: String)

earthquakesan

#!/bin/bash

export DOCKER_HOST="ubuntubox.ermilov.org"
export DOCKER_CERT_PATH="$(pwd)/.docker"
export DOCKER_CONFIG="$(pwd)/.docker"
export DOCKER_TLS_VERIFY=1