tonythor

import urllib3
import json

test_hook="https://hooks.slack.com/services/TEF2GKSV7/XXXXXXHPH/XXXXXXXXXXXqaFQy7O3"
payload = {'text':'lambda just sent a slack message to this channel'}

http = urllib3.PoolManager()

def lambda_handler(event, context):
    r=http.request("POST", test_hook,

tonythor

{
  "ModelName": "Image_Text",
  "ModelMetadata": {
    "Author": "",
    "DateCreated": "Feb 15, 2022, 09:52 PM",
    "DateLastModified": "Feb 15, 2022, 10:38 PM",
    "Description": "",
    "AWSService": "Amazon DynamoDB",
    "Version": "3.0"
  },

tonythor

# An example of how to use boto3 to hit AWS, get an appconfig
# configuration, then extract locations of secrets keys, and then
# look them up in AWS Secrets Manager.
# Should work fine in lambda if moved into a handler function. 

import boto3
import json
from botocore.exceptions import ClientError

class SecretsWrapper: 

tonythor

from airflow.utils.dates import days_ago
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.operators.python_operator import PythonOperator

# use this json when you trigger.
# it's here as a variable only for cut-and-paste formatting and to keep the demo portable
cutAndPasteJson = """
{
    "note": "For the passing variables dummy dag",

tonythor

FROM python:3.8

ARG AIRFLOW_VERSION=1.10.12
ARG AIRFLOW_USER_HOME=/usr/local/airflow
ARG AIRFLOW_DEPS=""
ARG PYTHON_DEPS=""
ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME}

COPY ./requirements.txt /requirements.txt

tonythor

import scala.annotation.tailrec
import scala.concurrent.duration.Duration
import scala.util.Random

// the function we'll run until true
def myFunction(): Boolean = {
  val rand = Random.nextInt()
  if (rand % 10 == 0) {
    print(s"${rand} is divisible by 10\n")
    true

tonythor

#!/bin/bash

# -> remember to run: dcos auth login first !! 
DCOS_API_TOKEN=$(dcos config show core.dcos_acs_token)
url="http://{marathon-domain}/service/{marathon zeppelin name}"
notebook="2E617JZX1" # $url/#/notebook/2E617JZX1
paragraph="20190916-164803_817623738"
#Note: to get paragraph ID, download notebook, open json and look for -> paragraphs -> Item [N] -> id.

curl --request GET -s -H "Content-Type: application/json" -H "Authorization: token=$DCOS_API_TOKEN" $url/api/notebook 

tonythor

package com.gimmesome.zeppelin

import com.softwaremill.sttp._
import scala.util.parsing.json.JSON

// case class ZeppelinConfig (instance: String, baseUrl: String, authLoginUrl: String, authUid: String, authPass: String)

// Usage:
//  import something.ZeppelinService
//  val notebook = "2E6T7JZX1"

tonythor

import org.apache.spark.sql.expressions.UserDefinedFunction
import org.apache.spark.sql.functions.udf 
// Usage: df.select(df.columns.map(c => emptyToNullUdf(col(c)).alias(c)): _*)
def emptyToNull(_str: String): Option[String] = {
  _str match {
    case d if (_str == null || _str.trim.isEmpty) => None
    case _ => Some(_str)
  }
}
val emptyToNullUdf = udf(emptyToNull(_: String))

tonythor

import com.github.mrpowers.spark.daria.sql.transformations
import scala.annotation.tailrec
// import other stuff related to spark

val DefaultReplacements = Map(
    "'" -> "\\'",
    "\"" -> "\\'",
    "," -> "\\,")

// if you wanted to pass in a list of columns, say all columns in a DF, you could replace like so.