elliottcordo

REGISTER 's3://caserta-bucket1/libs/elephant-bird-pig.jar'
REGISTER 's3://caserta-bucket1/libs/elephant-bird-core.jar'
REGISTER 's3://caserta-bucket1/libs/elephant-bird-hadoop-compat.jar'
REGISTER 's3://caserta-bucket1/libs/json-simple.jar'


business = LOAD 's3://caserta-bucket1/yelp-academic-dataset/yelp_academic_dataset_business.json'
  USING com.twitter.elephantbird.pig.load.JsonLoader('-nestedLoad');

business_cleaned = FOREACH business

elliottcordo

#MASTER=yarn-client /home/hadoop/spark/bin/pyspark

from pyspark.sql import SQLContext, Row
sqlContext = SQLContext(sc)

#------------------------------------------------
#load some users
lines=sc.textFile("s3://caserta-bucket1/yelp/in/users/users.txt")

parts = lines.map(lambda l: l.split(","))