tilakpatidar

Postgres snippet

create database test_db;

create table t_random as select s, md5(random()::text) from generate_Series(1,5000) s;

Pyspark snippet

In [1]: df=spark.read.jdbc(url="jdbc:postgresql://localhost:5432/test_db", table="t_random", properties={"driver": "org.postgresql.Driver"}).repartition(10)

tilakpatidar

#!/usr/bin/env bash
#https://www.datageekinme.com/setup/setting-up-my-mac-sqoop/

# Installation on mac
brew install sqoop
sudo mkdir /var/lib/accumulo

export ACCUMULO_HOME='/var/lib/accumulo'
export SQOOP_VERSION=1.4.6_1
export SQOOP_HOME=/usr/local/Cellar/sqoop/1.4.6_1/libexec

tilakpatidar

#!/usr/bin/env bash

#python
curl -X POST http://localhost:6066/v1/submissions/create --header "Content-Type:application/json;charset=UTF-8" --data '{
   "action":"CreateSubmissionRequest",
   "appArgs":[
      "/Users/tilak/jobs/test_job.py"
   ],
   "appResource":"file:/Users/tilak/jobs/test_job.py",
   "clientSparkVersion":"2.3.3",

tilakpatidar

Keybase proof

I hereby claim:

• I am tilakpatidar on github.
• I am tilakpatidar (https://keybase.io/tilakpatidar) on keybase.
• I have a public key whose fingerprint is B366 0F6B 48D9 5E12 D7DC 1487 FF74 B160 3F1C 7463

To claim this, I am signing this object:

tilakpatidar

# coding=utf-8

import findspark
from pandas.util.testing import assert_frame_equal

findspark.init()

import logging
import pytest

tilakpatidar

import spark.implicits._
import org.apache.spark.sql.SaveMode

val products = spark.sqlContext.read.format("jdbc").option("driver", "com.mysql.jdbc.Driver").option("dbtable", "products").option("user", "gobblin").option("password", "gobblin").option("url", "jdbc:mysql://localhost/mopar_demo").load()

scala> val newProducts = spark.sqlContext.read.format("orc").load("/Users/tilak/gobblin/mopar-demo/output/org/apache/gobblin/copy/user/tilak/pricing.products_1521799535.csv/20180325023900_append/part.task_PullCsvFromS3_1521945534992_0_0.orc")

scala> val reparitionedProducts = products.repartition(10)

val joined = newProducts.as("np").join(reparitionedProducts.as("op"), reparitionedProducts("sha") === newProducts("sha"), "left_outer")

tilakpatidar

import spark.implicits._
import org.apache.spark.sql.SaveMode
val products = spark.sqlContext.read.format("jdbc").option("driver", "com.mysql.jdbc.Driver").option("dbtable", "products").option("user", "gobblin").option("password", "gobblin").option("url", "jdbc:mysql://localhost/mopar_demo").load()

val newProducts = spark.sqlContext.read.format("orc").load("/Users/tilak/gobblin/mopar-demo/output/org/apache/gobblin/copy/user/tilak/pricing.products_1521799535.csv/20180325023900_append/part.task_PullCsvFromS3_1521945534992_0_0.orc")

val newnewProducts = newProducts.except(products)

val dfWriter = newnewProducts.write.mode(SaveMode.Append)
val connectionProperties = new java.util.Properties()

tilakpatidar

# ====================================================================
# PullCsvFromS3
# Pull CSV data from a directory S3 to MySQL
# ====================================================================

job.name=PullCsvFromS3
job.description=Pull CSV data from a directory S3 to MySQL
fs.uri=file:///

# Set working directory

tilakpatidar

# ====================================================================
# PullCsvFromS3
# Pull CSV data from a directory S3 to our local system
# ====================================================================

job.name=PullCsvFromS3
job.description=Pull CSV data from a directory S3 to our local system and write as AVRO files
fs.uri=file:///

# Set working directory

tilakpatidar

# ====================================================================
# PullCsvFromS3
# Pull CSV data from a directory S3 to our local system
# ====================================================================

job.name=PullCsvFromS3
job.description=Pull CSV data from a directory S3 to our local system
fs.uri=file:///

# Set working directory