speedtest.py

speedtest.py

import timeit
import pandas as pd
import requests

S3_URL = "https://s3-us-west-2.amazonaws.com/dssgtest01/crimedata.csv"
S3_PATH = "s3://dssgtest01/crimedata.csv"
LOCAL_EBS_PATH = "/mnt/vol-2a96ab12/crimedata.csv"
NFS_PATH = "/mnt/data1/SpeedTest/crimedata.csv"

#S3 Testing. using HTML
def s3_load_html():
        df = pd.read_csv(S3_URL)

#S3 Testing using S3
def s3_load_s3():
        df = pd.read_csv(S3_PATH)

#LOCAL EBS
def local_load():
        df = pd.read_csv(LOCAL_EBS_PATH)

#NFS Load
def nfs_load():
        df = pd.read_csv(NFS_PATH)

if __name__ == '__main__':
        print("S3 (HTML) Time:" + str(timeit.timeit("s3_load_html()", setup="from __main__ import s3_load_html",number=3)))
        print("S3 (S3) Time:" + str(timeit.timeit("s3_load_s3()", setup="from __main__ import s3_load_s3",number=3)))
        print("Local Time:" + str(timeit.timeit("local_load()", setup="from __main__ import local_load",number=3)))
        print("NFS Time:" + str(timeit.timeit("nfs_load()", setup="from __main__ import nfs_load",number=3)))