martin-g

Achieving warp speed with Rust

Contents:

• Number one optimization tip: don't
• Never optimize blindly
• Don't bother optimizing one-time costs
• Improve your algorithms
• CPU architecture primer
• Keep as much as possible in cache
• Keep as much as possible in registers

martin-g

apiVersion: scheduling.volcano.sh/v1beta1
kind: PodGroup
spec:
  minMember: 1
  minResources:
    cpu: "5"
    memory: "7Gi"
  priorityClassName: medium
  queue: my-queue

martin-g

#!/usr/bin/env bash

K8S_NAMESPACE="spark-on-k8s"
K8S_SERVICE_ACCOUNT_NAME="spark-account-name"
K8S_CLUSTER_ROLE="spark-on-k8s-cluster-role"

CURRENT_CONTEXT=(kubectl config view -o jsonpath="{.current-context}")

K8S_MASTER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$CURRENT_CONTEXT\")].cluster.server}")

martin-g

apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
  name: my-queue
spec:
  weight: 1
  capability:
    cpu: "6"
    memory: "8096Mi"

martin-g

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: high
value: 100
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: medium

martin-g

#!/usr/bin/env bash

DRIVER_CORES=1
DRIVER_MEM=2048m
EXECUTOR_INSTANCES=3
EXECUTOR_CORES=2
EXECUTOR_MEM=2048m
SPARK_HOME="/path/to/spark-3.3.0-SNAPSHOT-bin-with-volcano" # CHANGE 1

PATH="$JAVA_8_HOME/bin:$SPARK_HOME/bin:$PATH"

martin-g

#!/usr/bin/env bash

BASE_OS="openeuler/openeuler:20.03-lts-sp3"
SPARK_VERSION="3.3.0-SNAPSHOT"
SCALA_VERSION="scala_2.12"
DOCKERIMAGETAG="11-jre-slim"

cd $SPARK_HOME
./bin/docker-image-tool.sh \
  -t ${SPARK_VERSION}-${SCALA_VERSION}-${DOCKERIMAGETAG} \

martin-g

22/03/15 13:01:05 INFO LoggingPodStatusWatcherImpl: State changed, new state: 
	 pod name: spark-on-k8s-app-b336777f8daa93b6-driver
	 namespace: spark-on-k8s
	 labels: spark-app-selector -> spark-9e850f59cfa54f77bec1a2f30f4753a7, spark-role -> driver
	 pod uid: 48088478-9896-4019-b263-47866214cbb9
	 creation time: 2022-03-15T13:00:41Z
	 service account name: spark-account-name
	 volumes: spark-local-dir-1, spark-conf-volume-driver, kube-api-access-slmsc
	 node name: minikube
	 start time: 2022-03-15T13:00:41Z

martin-g

#!/usr/bin/env bash

DRIVER_CORES=1
DRIVER_MEM=2048m
EXECUTOR_INSTANCES=3
EXECUTOR_CORES=2
EXECUTOR_MEM=2048m
SPARK_HOME="$HOME/spark/spark-3.2.1-bin-hadoop3.2"

PATH="$JAVA_8_HOME/bin:$SPARK_HOME/bin:$PATH"

martin-g

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: my-kind-cluster-name
nodes:
- role: control-plane
- role: worker