samklr

# Copyright (C) 2006-2016 Amazon.com, Inc. or its affiliates.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#    http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is

samklr

resource "aws_instance" "int_tableau_linux" {
  key_name                    = "${var.key_name}"
  ami                         = "${data.aws_ami.int_tableau_linux.id}"
  instance_type               = "m5.4xlarge"
  iam_instance_profile        = "${aws_iam_instance_profile.int_tableau.id}"
  vpc_security_group_ids      = ["${aws_security_group.sgrp.id}"]
  associate_public_ip_address = false
  subnet_id                   = "${aws_subnet.subnet.id}"
  private_ip                  = "${var.dq_internal_dashboard_linux_instance_ip}"

samklr

kinit -t /etc/security/keytabs/c******-*****.keytab -k ****-info**t@NX.***.somewhere

klist

samklr

import spark.implicits._ 

  case class Row(id: Int, value: String)

  val r1 = Seq(Row(1, "A1"), Row(2, "A2"), Row(3, "A3"), Row(4, "A4")).toDS()
  val r2 = Seq(Row(3, "A3"), Row(4, "A4"), Row(4, "A4_1"), Row(5, "A5"), Row(6, "A6")).toDS()
  

samklr

#!/bin/sh

# vars
## EDITOR/VISUAL - what process to use to pick targets interactively
## ZK_WL - regex for zookeeper paths not to remove
## KAFKA_WL - regex for kafka topics not to remove
## MONGO_WL - regex for mongo item ids not to remove

# set -x

samklr

Keybase proof

I hereby claim:

• I am samklr on github.
• I am samklr_ (https://keybase.io/samklr_) on keybase.
• I have a public key ASAJAlW3njCb2s4F77DE8jY37PhD4uZVvuKUs6x71A15PAo

To claim this, I am signing this object:

samklr

https://blog.linuxserver.io/2017/07/17/i-deployed-a-plex-media-server-to-aws-because-why-not/

https://www.sqlchick.com/

https://fr.slideshare.net/Dataversity/data-lake-architecturehttps://www.slideshare.net/SparkSummit/07-blagoy-kaloferovhttps://www.slideshare.net/databricks/costbased-optimizer-in-apache-spark-22


https://jaceklaskowski.gitbooks.io/mastering-spark-sql/spark-sql-performance-tuning-groupBy-aggregation.htmlhttps://www.slideshare.net/databricks/lessons-from-the-field-episode-ii-applying-best-practices-to-your-apache-spark-applications-with-silvio-fiorito

samklr

import org.apache.spark.executor.TaskMetrics
import org.apache.spark.scheduler._

import scala.collection.mutable

class ValidationListener extends SparkListener {

  private val taskInfoMetrics = mutable.Buffer[(TaskInfo, TaskMetrics)]()
  private val stageMetrics = mutable.Buffer[StageInfo]()

samklr

import org.apache.spark.sql.{SaveMode, SparkSession}
case class HelloWorld(message: String)

def main(args: Array[String]): Unit = {
    
        // Creation of SparkSession
        val sparkSession = SparkSession.builder()
          .appName("example-spark-scala-read-and-write-from-hive")
          .config("hive.metastore.warehouse.dir", params.hiveHost + "user/hive/warehouse")
          .enableHiveSupport()

samklr

Error handling pitfalls in Scala

There are multiple strategies for error handling in Scala.

Errors can be represented as [exceptions][], which is a common way of dealing with errors in languages such as Java. However, exceptions are invisible to the type system, which can make them challenging to deal with. It's easy to leave out the necessary error handling, which can result in unfortunate runtime errors.