chathurawidanage

from cProfile import label
import re
from datetime import datetime

host_mem = 73
dev_mem = 16

def plot(worker = '0'):
    print("Plotting worker", worker)
    logs = open("theseus_log/batch_log."+worker+".log")

chathurawidanage

from theseus import TheseusContext
tc = TheseusContext(config_options={
    "EXECUTOR_THREADS": 1
}, output_type="cudf")

tc.create_table('ex', 'example.parquet')

tc.create_table('nation', 'nation.parquet')
tc.create_table('region', 'region.parquet')

chathurawidanage

import types
import pyarrow.parquet as pq

import pyarrow as pa
import pandas as pd
import numpy as np

data = np.array(np.random.random_sample((5,)), dtype=np.float32)
data2 = np.array(np.random.random_sample((5,)), dtype=np.float32)
data3 = np.array(np.random.random_sample((5,)), dtype=np.float32)

chathurawidanage

apiVersion: kubeflow.org/v1alpha2
kind: MPIJob
metadata:
  name: cylon-examples
spec:
  slotsPerWorker: 1
  cleanPodPolicy: Running
  mpiReplicaSpecs:
    Launcher:
      replicas: 1

chathurawidanage

from pycylon import DataFrame, read_csv, CylonEnv
from pycylon.net import MPIConfig

env = CylonEnv(config=MPIConfig())
print("Hello from Worker : ", env.rank)

df1 = read_csv("file1.csv", slice=True)
df1.set_index([0])

df2 = read_csv("file2.csv", slice=True)

chathurawidanage

from pycylon import DataFrame, read_csv, CylonEnv
from pycylon.net import MPIConfig

env = CylonEnv(config=MPIConfig())
print("Hello from Worker : ", env.rank)

df1 = read_csv("file1_"+str(env.rank)+".csv")
df1.set_index([0])

df2 = read_csv("file2_"+str(env.rank)+".csv")

chathurawidanage

from pycylon import DataFrame, read_csv, CylonEnv
from pycylon.net import MPIConfig

env = CylonEnv(config=MPIConfig())
print("Hello from Worker : ", env.rank)

df1 = read_csv("file1.csv")
df1.set_index([0])

df2 = read_csv("file2.csv")

chathurawidanage

from pycylon import DataFrame, read_csv

df1 = read_csv("file1.csv")
df1.set_index([0])

df2 = read_csv("file2.csv")
df2.set_index([0])

join = df1.merge(df2, left_on=[0], right_on=[0])
print(join)

chathurawidanage

// #include "config.h"
// #include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include <algorithm>
#include <stdint.h>
#include <getopt.h>
// #include "Uncompress.h"

chathurawidanage

#include <cstdlib>
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <queue>
#include <list>
#include <stdint.h>
#include <getopt.h>