As a reminder, here are the components in play to run an application:
- The cluster:
- Spark Master: coordinates the resources
- Spark Workers: offer resources to run the applications
- The application:
{"err":["Välkommen","Välkommen till Växla upp 2 !"]} |
[user] | |
name = Sunny | |
email = Sunny | |
[diff] | |
tool = vimdiff | |
[difftool] | |
prompt = true |
# I was having problems to create docker images for spark-on-k8s using docker-image-tool.sh in spark (it always gives spark-r). | |
# Thus here is the docker file to creat for running pyspark. | |
# The image is available on sunnybingome/spark8s:pyspark240py368 with spark 2.4.0 | |
# For clie-mode (interactive mode), please use pipenv to ensure py version. The image is in using py 3.6(.8). | |
# Sunny modified from spark-2.4.0-bin-hadoop2.7/kubernetes/dockerfiles/spark/Dockerfile | |
# | |
# Licensed to the Apache Software Foundation (ASF) under one or more | |
# contributor license agreements. See the NOTICE file distributed with |
This is a gist of collection of simple spark tests for Jupyter and spark-submit.
import socket | |
print([l for l in ([ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] | |
if not ip.startswith("127.")][:1], [[(s.connect(('8.8.8.8', 53)), | |
s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, | |
socket.SOCK_DGRAM)]][0][1]]) if l][0][0]) | |
# limit: only one IP, not getting multi-NIC IPs |
# Import module from previous step | |
Import-Module -Name posh-git | |
function Test-Administrator { | |
$user = [Security.Principal.WindowsIdentity]::GetCurrent(); | |
(New-Object Security.Principal.WindowsPrincipal $user).IsInRole([Security.Principal.WindowsBuiltinRole]::Administrator) | |
} | |
function mklink { cmd /c mklink $args } |
# pip2 | |
absl-py==0.2.2 | |
adium-theme-ubuntu==0.3.4 | |
astor==0.6.2 | |
backports.functools-lru-cache==1.5 | |
backports.shutil-get-terminal-size==1.0.0 | |
backports.weakref==1.0.post1 | |
bleach==1.5.0 | |
certifi==2018.4.16 | |
chardet==3.0.4 |
apt-get purge cuda && \ | |
apt-get purge libcudnn6 && \ | |
apt-get purge libcudnn6-dev && \ | |
wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_9.0.176-1_amd64.deb && \ | |
wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/libcudnn7_7.0.5.15-1+cuda9.0_amd64.deb && \ | |
wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/libcudnn7-dev_7.0.5.15-1+cuda9.0_amd64.deb && \ | |
wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/libnccl2_2.1.4-1+cuda9.0_amd64.deb && \ | |
wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/libnccl-dev_2.1.4-1+cuda9.0_amd64.deb && \ | |
dpkg -i cuda-repo-ubuntu1604_9.0.176-1_amd64.deb && \ | |
dpkg -i libcudnn7_7.0.5.15-1+cuda9.0_amd64.deb && \ |
var links = document.links; | |
for (var i = 0, linksLength = links.length; i < linksLength; i++) { | |
if (links[i].hostname != window.location.hostname) { | |
links[i].target = '_blank'; | |
} | |
} |