Skip to content

Instantly share code, notes, and snippets.

Avatar

João Antonio Ferreira joao-parana

View GitHub Profile
View Workflow01UDAF.scala
// UserDefinedAggregateFunction is the contract to define
// user-defined aggregate functions (UDAFs)
class MyCountUDAF extends UserDefinedAggregateFunction {
// Este método abaixo define pode ser invocado apenas assim: inputSchema(0)
// Isto é feito via inversão de dependência pelo Spark
// o retorno é um objeto StructField assim:
// StructField("id", LongType, true, {})
// o objeto StructField é do pacote org.apache.spark.sql.types
override def inputSchema: StructType = {
new StructType().add("id", LongType, nullable = true)
View theme.md

<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAmUAAAIjCAIAAADN7PsBAAAYMWlDQ1BJQ0MgUHJvZmlsZQAAWIWVeQVUVN+3/7mTzDAM3d0lzZDS3d0IDD10p0WIlCIIKAIqqCCoYBEiFoKAIoIKGIiEIqWCCoKAvEvo9/e+77/+b72z1rn3s/bZZ5/PvvvUngGAU5EcERGCoAMgNCwmytZIl8/ZxZUPOw4ggAYEIAhIZJ/oCB1ra3MAlz/v/16WBmFtuLyQ2rT1P9v/v4Xe1y/aBwDIGsbevtE+oTC+CQBKySciKgYA9BwsF4yPiYAxBmYJmKJggjAW2sQB21hlE3tvY/MtHXtbPRh7AUBFIJOjAgAgbvLii/MJgO0Qc+E2hjBfShisehbGmj6BZF8AOEZhnV2hoeEw5iTAWMz7P+wE/Deb3n9tkskBf/G2L1uFSp8SHRFCTvw/fo7/vYSGxP4ZQxCuhMAoY9tNnze/W3C42SaGuUNdYd6WVjBmgPFLiu+W/iaeDIw1dtjRX/aJ1oO/GWABAEHwJeubwZgLxgKxwQ46O1iTHLXVF9ZHuCYF2jtt20eERYXb7thHJIWFWJrv2MkN9DP5gyv9og3s/uj4UwxNYAzHENFMiTGx37HZFUdxtIQxEcbD0cF2Zjt9Z5IC9Sz/jhVru8kZjjkShEb/8QUp5B9laLutj1QKpJhY7sjNYwLtjbf7Ij18yFsc2GAc5BftbP6Hj6+fvsE2H2SqX5jDDk9kQUSMru2O/vmIEOsdfWSrX4jRplwAxr3RcXZ/+s7HwJNt2xcUCCKbWm+Pi2KKiLG23+aG4gPmQA/oAz4QC1dvEA6CAKV3rmkO/GkxBGQQBQKAH5Dakfzp4bTVEgY/7UAS+AwjPxD9t5/uVqsfiIPl63+l208p4L/VGrfVIxhMwjgUxYHSRKmjzOGnNlzlUSoo1T/9+Gj/jIoxwOhjjDGGGHFPSmrUv+zyAR/YgxC4RgEz+O0He7XJIewP

View diagrama-W1.svg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View W1.scala
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.SparkSession.Builder
import org.apache.spark.SparkContext
import org.apache.log4j.{Level, Logger}
// A sparkSession é provida pelo proprio Spark Shell
// O nivel de log também já é configurado pela Spark Shell
def boolean_udf_wrapper(a:String, b:String, t:Any): Boolean = { true }
def string_udf_wrapper(a:String, b:String, t:Any): String = { "••••" }
import org.apache.spark.sql.functions.expr
import org.apache.spark.sql.functions.sum
View Workflow01UDFs.scala
def happyEmployees(salary: Int) => salary > 2200
def smartTextCase(name: String) => name.toUpperCase()
View kubernetes-dnd.sh
# Usando o Kubernetes com Docker in Docker (DIND)
sudo mkdir -p /usr/local
cd /usr/local
sudo mkdir dind-cluster
cd dind-cluster/
sudo chmod o+w .
ls -lat .. | head
# wget https://cdn.rawgit.com/kubernetes-sigs/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.10.sh
curl -O https://cdn.rawgit.com/kubernetes-sigs/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.10.sh
@joao-parana
joao-parana / .block
Created Mar 28, 2018 — forked from mbostock/.block
Chord Diagram
View .block
license: gpl-3.0
height: 960
border: no
@joao-parana
joao-parana / map-pushdow.sc
Last active Apr 20, 2018
Using CatalystExtension Points in Spark
View map-pushdow.sc
// Este script é para rodar no Ammonite.
// Crie o arquivo catalyst_04.sc com este conteúdo
// Dentro da shell REPL do Ammonitem, você deve invocar assim:
// import $file.catalyst_04, catalyst_04._
//
// Mas antes execute estes tres comandos abaixo
// import coursier.MavenRepository
// interp.repositories() ++= Seq(MavenRepository("file:/Users/admin/.m2/repository"))
// import $ivy.`org.apache.spark::spark-sql:2.3.0`
@joao-parana
joao-parana / index.html
Last active Dec 19, 2017
SOMA 02 - data binding
View index.html
<!DOCTYPE html>
<html>
<head>
<meta name="description" content="data-binding">
</head>
<body>
<script src="https://d3js.org/d3.v4.min.js" charset=" utf-8"></script>
<script>
var selector = d3.select('body')
.selectAll('span')
@joao-parana
joao-parana / index.html
Last active Dec 19, 2017
Exemplo simples com select
View index.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="description" content="SOMA 01 - Hello World">
<title>SOMA 01 - Hello World</title>
</head>
<body>
<div id="page"></div>
<script src="https://d3js.org/d3.v4.min.js" charset="utf-8"></script>