rafaelfelix

#!/usr/bin/env python
from __future__ import print_function

import argparse
import os
import shutil
import sys

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Native Instruments uninstaller util for Mac (see: https://support.native-instruments.com/hc/en-us/articles/210291865-How-to-Uninstall-Native-Instruments-Software-from-a-Mac-Computer)")

rafaelfelix

#!/usr/bin/env python
import datetime
from requests_aws4auth import AWS4Auth
from elasticsearch import Elasticsearch, RequestsHttpConnection
import boto3

host = '' # For example, search-my-domain.region.es.amazonaws.com
region = '' # For example, us-west-1
service = 'es'
credentials = boto3.Session().get_credentials()

rafaelfelix

#!/bin/bash
IMAGE_REPO=$(kubectl get svc registry-docker-registry -o=jsonpath='{.spec.clusterIP}'):5000

cat > clustermode-podspec.yaml << EOF
apiVersion: v1
kind: Pod
metadata:
  name: spark-submit-example
spec:
  serviceAccountName: spark

rafaelfelix

#!/bin/bash

# preventing issue https://issues.apache.org/jira/browse/SPARK-28921 from happening
rm -f ./spark/jars/kubernetes*.jar

curl -Lo ./spark/jars/kubernetes-model-4.4.2.jar https://repo1.maven.org/maven2/io/fabric8/kubernetes-model/4.4.2/kubernetes-model-4.4.2.jar
curl -Lo ./spark/jars/kubernetes-model-common-4.4.2.jar https://repo1.maven.org/maven2/io/fabric8/kubernetes-model-common/4.4.2/kubernetes-model-common-4.4.2.jar
curl -Lo ./spark/jars/kubernetes-client-4.4.2.jar https://repo1.maven.org/maven2/io/fabric8/kubernetes-client/4.4.2/kubernetes-client-4.4.2.jar

rafaelfelix

apiVersion: v1
kind: ServiceAccount
metadata:
  name: spark
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: spark-cluster-role
rules:

rafaelfelix

$ kubectl get pods -w
NAME                                        READY   STATUS    RESTARTS   AGE
registry-docker-registry-59dbb85998-rhl5j   1/1     Running   0          25m
spark-pi-1573667443103-exec-1               0/1     Pending   0          0s
spark-pi-1573667443103-exec-1               0/1     Pending   0          0s
spark-pi-1573667443103-exec-1               0/1     ContainerCreating   0          0s
spark-pi-1573667443658-exec-2               0/1     Pending             0          0s
spark-pi-1573667443658-exec-2               0/1     Pending             0          0s
spark-pi-1573667443103-exec-1               1/1     Running             0          18s
spark-pi-1573667443103-exec-1               1/1     Terminating         0          31s

rafaelfelix

Using properties file: null
Parsed arguments:
  master                  k8s://https://192.168.99.105:8443
  deployMode              client
  executorMemory          null
  executorCores           null
  totalExecutorCores      null
  propertiesFile          null
  driverMemory            null
  driverCores             null

rafaelfelix

#!/bin/bash
IMAGE_REPO=$(kubectl get svc registry-docker-registry -o=jsonpath='{.spec.clusterIP}'):5000
export SPARK_HOME=$(pwd)/spark

$SPARK_HOME/bin/spark-submit \
    --master k8s://https://$(minikube ip):8443 \
    --conf spark.kubernetes.container.image=$IMAGE_REPO/spark \
    --class org.apache.spark.examples.SparkPi \
    $SPARK_HOME/examples/jars/spark-examples_2.11-2.4.4.jar

rafaelfelix

#!/bin/bash
./spark/bin/docker-image-tool.sh -r $(minikube ip):30000 -t latest build
# building will take a bit of time. then:
./spark/bin/docker-image-tool.sh -r $(minikube ip):30000 -t latest push

rafaelfelix

#!/bin/bash
minikube ip