save_rdd_to_csv.py

import re

def extract_url(cid, id32):
    chop_id = '/'.join(re.findall('..', '{:0>10}'.format(cid))[0:4])
    size = '240'
    return (str(cid), 'https://t3.ftcdn.net/jpg/{}/{}_F_{}_{}_NW.jpg'.format(chop_id, size, cid, id32))

def extract_tags(rdd_record):
    j = json.loads(rdd_record)
    tags = [x.split('^')[0] for x in j['k']['eksrg']]
    cid, url = extract_url(j['cid'], j['id32'])
    return cid, url,tags 

first_100_img_tags = first_100_rows.map(lambda record: extract_tags(record))

def toCSVLine(data):
  return ','.join(str(d) for d in data)

lines = first_100_img_tags.map(toCSVLine)

lines.repartition(1).saveAsTextFile("s3://psriniva/img_cid_url_tags_100.csv")