Skip to content

Instantly share code, notes, and snippets.

View Spark – Redshift: AWS Roles to the rescue.py
import requests
from requests.exceptions import HTTPError, Timeout
from pyspark import SparkConf
from pyspark import SparkContext
from pyspark.sql import SQLContext
DEFAULT_INSTANCE_ROLE = 'STAGING_EMR_CLUSTER_ROLE'
conf = SparkConf().setAppName(opts.get('app_name'))
View Spark-streaming-wordcount.scala
package com.berlinsmartdata.spark.streaming
import org.apache.spark._
import org.apache.spark.streaming._
import org.apache.spark.streaming.StreamingContext._
object Sample {
def main(args: Array[String]):Unit = {
val numReceivers = 2 //note: has to be >= 2
View tensor_flow_graph_snippet.py
batch_size = 16
patch_size = 5
depth = 16
num_hidden = 64
graph = tf.Graph()
with graph.as_default():
# Input data: train, test and validation set
tf_train_dataset = tf.placeholder(
@diogoaurelio
diogoaurelio / dummy_scala_thread_deadlock_avoid_example.scala
Last active Sep 28, 2016
dummy_scala_thread_deadlock_avoid_example.scala
View dummy_scala_thread_deadlock_avoid_example.scala
package parallelProg.week1
object Example0501 extends App {
// Resolving deadlocks: acquire resources in the same order
class Account(var amount:Int = 0, val name:String) {
private val x = new AnyRef {}
private var uidCount = 0L
val uid = getUniqueId()
View ScalaJHttpRequester.scala
import scalaj.http._
import scala.concurrent.{ExecutionContext, Future}
/**
* Trait for generic scalaj HTTP requests
*
*/
trait ScalaJHttpRequester {
def getRequest(address:String,
View PlayWsRequester.scala
import com.ning.http.client.{AsyncHttpClientConfigBean, AsyncHttpClientConfig}
import play.api.libs.ws.ning.NingWSClient
import play.api.libs.ws.ning.NingAsyncHttpClientConfigBuilder
import play.api.libs.ws.ssl.{SSLLooseConfig, SSLConfig}
import play.api.libs.ws.{WSClientConfig, DefaultWSClientConfig, WSResponse}
import scala.concurrent.{ExecutionContext, Future}
import play.api.libs.json._
trait PlayWsRequester {
View AkkaHttpApi.scala
import akka.event.{Logging, LoggingAdapter}
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.http.scaladsl.server.Directives._
import akka.stream.Materializer
import akka.stream.scaladsl.{Flow, Sink, Source}
import com.typesafe.config.Config
import com.typesafe.config.ConfigFactory
import scala.concurrent.{ExecutionContextExecutor, Future}
import spray.json.DefaultJsonProtocol
View akkaDemo.scala
import java.util.concurrent.TimeUnit
import akka.actor.{Actor, ActorSystem, Props}
import scala.concurrent.Await
import scala.concurrent.duration.Duration
View tensorflow-getting-started.py
import tensorflow as tf
a = tf.constant(1)
b = tf.constant(2)
c = a + b
with tf.Session() as session:
result = session.run(c)
print("Tensorflow supports simple summation with op: 'a + b = {}'".format(result))
# Tensorflow supports simple summation with op: a + b = 3
View getting_started_tensorflow_3.py
import tensorflow as tf
x = tf.Variable(3, name="x", dtype=tf.int16)
y = tf.Variable(4, name="y", dtype=tf.int16)
a = tf.constant(5, dtype=tf.int16)
b = tf.constant(10, dtype=tf.int16)
c = tf.constant(15, dtype=tf.int16)
op1 = x*x*y