Skip to content

Instantly share code, notes, and snippets.

@diogoaurelio
diogoaurelio / min-char-rnn.py
Created Nov 21, 2018 — forked from karpathy/min-char-rnn.py
Minimal character-level language model with a Vanilla Recurrent Neural Network, in Python/numpy
View min-char-rnn.py
"""
Minimal character-level Vanilla RNN model. Written by Andrej Karpathy (@karpathy)
BSD License
"""
import numpy as np
# data I/O
data = open('input.txt', 'r').read() # should be simple plain text file
chars = list(set(data))
data_size, vocab_size = len(data), len(chars)
View kube_config.yaml
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: <certificateAuthority.data from describe-cluster>
server: <endpoint from describe-cluster>
name: <cluster-name>
contexts:
- context:
cluster: <cluster-name>
user: aws
@diogoaurelio
diogoaurelio / db_utils_pg8000.py
Created Oct 24, 2018
Sample utility functions for using pg8000 to connect with Postgres
View db_utils_pg8000.py
"""
Utils to interact with DB using pg8000 library
Note: assumes py3.5+
[Example usage]
# OPTIONALLY retrieve DB password from AWS SSM
import boto3
View pyspark_fixtures.py
"""
Utilities common to all tests using spark
"""
import pytest
from pyspark.sql import SparkSession
from pyspark import SparkContext, SparkConf
import logging
@diogoaurelio
diogoaurelio / pyspark_demo_app_unit_test.py
Created Oct 21, 2018
PySpark demo Application example unit test
View pyspark_demo_app_unit_test.py
from tests.test_utils.test_spark import spark_session
from pyspark.sql import DataFrame, Row, SparkSession
from pyspark.sql.functions import col
from src.job import amount_spent_udf
def test_amount_spent_udf(spark_session: SparkSession) -> None:
input_df = spark_session.createDataFrame([
Row(customer_name="Geoffrey", date="2016-04-22", category="Foo", product_name="Bar", quantity=1, price=2.00),
View pyspark_demo_app.py
# create the general function
def _amount_spent(quantity: int, price: float) -> float:
"""
Calculates the product between two variables
:param quantity: (float/int)
:param price: (float/int)
:return:
(float/int)
"""
return quantity * price
View put_ssm_parameter.py
my_string = """
your string \n seperated \n by \n new \n lines.
"""
account_id = '12345678910'
region = 'eu-west-1'
parameter_name = 'some-secret-name'
key_id = 'your-key-id'
View redshift_in_flight_queries.sql
# Redshift - check currently running queries; useful for misbehaving queries;
SELECT
userid
, query
, pid
, starttime
, left(text, 100) as query
FROM stv_inflight;
View lambda_permissions.tf
################################################################################
# AWS Lambda IAM Policy document definitions
################################################################################
data "aws_iam_policy_document" "this" {
statement {
effect = "Allow"
actions = [
"s3:GetBucketLocation",
@diogoaurelio
diogoaurelio / locals.tf
Last active Sep 16, 2018
lambda_locals_env_vars.tf
View locals.tf
################################################################################
# Locals used for different Lambdas Environmental Variables
################################################################################
locals {
redshift_loader_lambda_env_vars = {
ENVIRONMENT = "${var.environment}"
REGION = "${var.aws_region}"
IAM_ROLE = "${var.redshift_data_loader_lambda_iam_role}"