sujee

#!/usr/bin/env python

## This CNN network to identify MNIST dataset

import time
import random
import numpy as np
from pprint import pprint
import os, sys

sujee

## Step 3 : encode the indexes into a vector
from pyspark.ml.feature import OneHotEncoder

encoder = OneHotEncoder(inputCols=["statusIndex"], outputCols=["statusVector"], dropLast=False)
encoded = encoder.fit(indexed).transform(indexed)
encoded.show()

# View dense vectors in pandas
encoded_pd = encoded.toPandas()
print(encoded_pd)

sujee

Loading models...
Loaded model 'transformer-qa' in 31,693.4 milli seconds
Loaded model 'bidaf-model' in 1,633.8 milli seconds
/Users/sujee/opt/anaconda3/envs/teachbot-nlp/lib/python3.7/site-packages/torch/nn/modules/container.py:435: UserWarning: Setting attributes on ParameterList is not supported.
  warnings.warn("Setting attributes on ParameterList is not supported.")
Loaded model 'bidaf-elmo-model' in 13,811.0 milli seconds

quesion:  Who stars in The Matrix?
   model transformer-qa predicted in 794.4 milli seconds
   answer:  Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss, Hugo Weaving, and Joe Pantoliano

sujee

sujee

#!/bin/bash

## invoke with '-d' for dev mode
## this will mount utils directory from host for live debugging

port=2000

while getopts 'dp:' OPTION; do
  case "$OPTION" in
    d)

sujee

#!/bin/bash

if [ -z "$1" ] ; then
    echo "Usage:  $0    <image name>    [optional args for docker image]"
    echo "Missing Docker image id.  exiting"
    exit 1
fi

#image_id="$1"
#shift

sujee

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>

sujee

<?xml version="1.0"?>
<configuration>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>localhost</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>

sujee

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
        <description>single node, 1 copy</description>
    </property>
</configuration>

sujee

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:8020</value>
        <description>
            this replaces Hadoop 1 property : fs.default.name
        </description>
    </property>