rao-abdul-mannan

(function () {
	angular
		.module('Toastar',[])
		.factory('Toastar',Toastar);

	Toastar.$inject = [];
	function Toastar() {
		var factory = {
			toastSettings:toastSettings,
			setSuccessToast:setSuccessToast,

rao-abdul-mannan

#!/usr/bin/env bash
# MIT © Sindre Sorhus - sindresorhus.com

# git hook to run a command after `git pull` if a specified file was changed
# Run `chmod +x post-merge` to make it executable then put it into `.git/hooks/`.

changed_files="$(git diff-tree -r --name-only --no-commit-id ORIG_HEAD HEAD)"

check_run() {
	echo "$changed_files" | grep --quiet "$1" && eval "$2"

rao-abdul-mannan

### This gist explains how to connect jupyterhub with Spark2 on CDH 5.13 Cluster. 
Following below instructions, Spark can be configured with Jupyterhub on any cluster, standalone or locally etc

- Install jupyterhub by following instructions on official repo https://github.com/jupyterhub/jupyterhub 
- Once installed, before configuring spark2 kernel first locate the jupyter's kernels directory. 
  On centos7, its under /usr/share/jupyter/kernels/
- Assuming all jupyterhub kernel's are in /usr/share/jupyter/kernels/ 
  mkdir /usr/share/jupyter/kernels/pyspark2
- Create pyspark2 kernel 
  touch /usr/share/jupyter/kernels/pyspark2/kernel.json

rao-abdul-mannan

from pyspark.sql.types import *
from pyspark.sql.functions import *

#Flatten array of structs and structs
def flatten(df):
   # compute Complex Fields (Lists and Structs) in Schema   
   complex_fields = dict([(field.name, field.dataType)
                             for field in df.schema.fields
                             if type(field.dataType) == ArrayType or  type(field.dataType) == StructType])
   while len(complex_fields)!=0: