Skip to content

Instantly share code, notes, and snippets.

View matheus-rossi's full-sized avatar
🎯
Data Engineer

Matheus Rossi matheus-rossi

🎯
Data Engineer
View GitHub Profile
import string, yaml
def load_yaml(file_path: str, context: dict = None):
def string_constructor(loader, node):
t = string.Template(node.value)
value = t.substitute(context)
return value
l = yaml.SafeLoader
l.add_constructor('tag:yaml.org,2002:str', string_constructor)
import sys
# List comprehension
list_comprehension = [i for i in range(10_000_000)]
print(f"List comprehension memory: {sys.getsizeof(list_comprehension) / (1024 * 1024)} MB")
# Yield generator
def generator():
for i in range(10_000_000):
yield i
@matheus-rossi
matheus-rossi / spark_tips_01.py
Created February 21, 2024 18:26
spark_tips_01
from pyspark.sql import SparkSession
spark = (
SparkSession
.builder
.appName("spark_parameterized_queries")
.getOrCreate()
)
##### Criando dois datasets de teste #####
apiVersion: v2
name: datahub-prerequisites
description: A Helm chart for packages that Datahub depends on
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
version: 0.0.14
dependencies:
- name: elasticsearch
version: 7.17.3
const configPossibilities = [ [ 1 ], [ 1, 2 ], [ 1, 2, 3, 4 ] ]
const configPossibilities2 = {
ID_LINHA: [ 1 ],
ID_IMPLEMENTO: [ 1 , 2 ],
ID_COMPOSICAO: [ 1 , 2 , 3 , 4 ]
}
const desiredResult = [
[1, 1, 1 ],
// Somando valores de arrays (.reduce)
const numeros2 = [1, 2, 3]
function soma (x, y) {
return x + y
}
const somaDosNumeros2 = numeros2.reduce(soma)
console.log(somaDosNumeros2) // 6
// Objeto Inicial
const refeicaoArray = [
{ id: 1, descricao: 'Café da Manhã', calorias: 420 },
{ id: 2, descricao: 'Almoço', calorias: 580 }
]
// Removendo item no array refeicoes (.filter)
function filtro (refeicao) {
if (refeicao.id === 2) {
return true
// array inicial
const refeicaoArray = [
{ id: 1, descricao: 'Café da Manhã', calorias: 420 },
{ id: 2, descricao: 'Almoço', calorias: 580 }
]
const novaRefeicao = {
id: 3,
descricao: 'Café da Tarde',
calorias: 280
// Atualizando item no array (.map)
const numeros = [1, 2, 3]
function dobrar (num) {
return num * 2
}
const dobrarEs6 = num => num *2
const numerosVezesDois = numeros.map(dobrarEs6)
// Arrays Iniciais
const refeicaoArray = [
{ id: 1, descricao: 'Café da Manhã', calorias: 420 },
{ id: 2, descricao: 'Almoço', calorias: 580 }
]
const novaRefeicao = {
id: 3,
descricao: 'Café da Tarde',
calorias: 280