dardanxhymshiti

python setup.py --version

dardanxhymshiti

from setuptools import setup, find_packages

__version__ = '1.0.0'

setup(
    name='pyspark_seed', 
    version=__version__, 
    packages=find_packages(), 
    install_requires=['pyspark', 'boto3']
)

dardanxhymshiti

python3 -m venv venv

. ./venv/bin/activate

pip install -e .

dardanxhymshiti

def process(spark, input_path, output_path, save_mode='append'):
    # read data
    df = spark.read.parquet(input_path)

    # processing
    pass

    # output
    df.write.parquet(output_path, save_mode=save_mode)

dardanxhymshiti

import boto3
from os.path import join


def lambda_handler(event, context):
    emr = boto3.client('emr')
    version = 'latest'
    main_path = join('s3://<artifacts-bucket-name>', version, 'main.py')
    modules_path = join('s3://<artifacts-bucket-name>', version, 'module_seed.zip')

dardanxhymshiti

from module_seed.jobs import \
    job_one_seed,\
    job_two_seed
from module_seed.utils.logger_utils import get_logger
from module_seed.utils.spark_utils import create_spark_session


jobs = {
    'job_one_seed': job_one_seed.process,
    'job_two_seed': job_two_seed.process

dardanxhymshiti

import ast
import sys
from module_seed.run import run


if __name__ == '__main__':
    str_parameters = sys.argv[1]
    parameters = ast.literal_eval(str_parameters)
    run(parameters)

dardanxhymshiti

list_of_fruits = ['apple', 'orange', 'pear', 'cherry', 'banana']
list_of_scores = [80, 98, 50, 55, 100]

tuple_of_person_info = ('John', 'Doe', 20, 'Denver', 'Colorado')
tuple_of_car_info = ('BMW', 'X5', 5000, 'Black', True)

dardanxhymshiti

Ant-Man, Aquaman, Asterix, The Atom, 
Batgirl, Batman, Batwoman, 
Black Panther, Captain Marvel, Catwoman, 
Daredevil, The Defenders, Doc Savage, 
Elektra, Fantastic Four, Green Arrow, 
Guardians Of The Galaxy, Hawkeye, Hellboy, 
Iron Fist, Iron Man, Marvelman, 
The Shadow, Spider-Man, Sub-Mariner, 
Superman, Teenage Mutant Ninja Turtles, Thor, 
Wolverine, Wonder Woman, X-Men, Zatara

dardanxhymshiti

Paris, Doha, Amsterdam, Venice, Koyoto, Barcelona, 
Athens, Sydney, San Miguel de Allende, Havana, Beirut, 
Cape Town, Budapest, Buenos Aires, Lisbon, Luang Prabang, 
Florence, Istanbul, Hong Kong, Copenhagen, Jerusalem, 
Krakow, Bruges, Busan, Dubrovnik, Cartagena, 
Edinburgh, Québec City, Hamburg, Jaipur, Queenstown, 
Muscat, London, Chefchaouen, Charleston, Seville, 
New York, Isfahan, Rio de Janeiro, Rome, Shanghai, 
St Petersburg, Vancouver, Tallinn, Vienna, Pristina, 
Singapore, Prague, Dublin, Lucerne, Quito