davehowell

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types._
import org.slf4j.{Logger, LoggerFactory}

object Main {
  val logger: Logger = LoggerFactory.getLogger(this.getClass)

  private lazy val sparkConf: SparkConf = new SparkConf()
    .setMaster("local[*]")

davehowell

#!/bin/bash

# Add these functions to your bash_profile or bash_rc
# Dependencies:
#   - awscli - AWS command line client
#   - jq - Command-line JSON processor

function cfn-validate-template() {
for template in $@
do

davehowell

# Totally ~plagiarised~ inspired from the forked file by Vince Veselosky in this gist, extended to work with jsonlines format 
# and skipping the S3 write so the serialisation can be tested without mocking S3

import re
import io
import json
import gzip
import dateutil.parser
from json import JSONEncoder
from pprint import pprint

davehowell

PSQL

Magic words:

psql -U postgres

Some interesting flags (to see all, use -h or --help depending on your psql version):

• -E: will describe the underlaying queries of the \ commands (cool for learning!)
• -l: psql will list all databases and then exit (useful if the user you connect with doesn't has a default database, like at AWS RDS)