Skip to content

Instantly share code, notes, and snippets.

Will Jones drj42

Block or report user

Report or block drj42

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
@drj42
drj42 / types_and_roles_demo.sql
Created Sep 5, 2018 — forked from levlaz/types_and_roles_demo.sql
Create Types and Roles If Not Exist in PostgreSQL
View types_and_roles_demo.sql
BEGIN;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'task_status') THEN
create type task_status AS ENUM ('todo', 'doing', 'blocked', 'done');
END IF;
END
$$;
CREATE TABLE IF NOT EXISTS
@drj42
drj42 / postgres_queries_and_commands.sql
Last active Aug 29, 2018 — forked from rgreenjr/postgres_queries_and_commands.sql
Useful PostgreSQL Queries and Commands
View postgres_queries_and_commands.sql
-- long running queries
SELECT
pid,
now() - pg_stat_activity.query_start AS duration,
query,
state
FROM pg_stat_activity
WHERE (now() - pg_stat_activity.query_start) > interval '5 minutes'
-- show running queries (9.2)
@drj42
drj42 / kube-registry.yaml
Created Nov 22, 2017 — forked from coco98/kube-registry.yaml
Docker registry on minikube
View kube-registry.yaml
apiVersion: v1
kind: ReplicationController
metadata:
name: kube-registry-v0
namespace: kube-system
labels:
k8s-app: kube-registry
version: v0
spec:
replicas: 1
@drj42
drj42 / async_worker_pool.py
Created Jan 30, 2017 — forked from thehesiod/async_worker_pool.py
Asynchronous Worker Pool
View async_worker_pool.py
import asyncio
from datetime import datetime, timezone
import os
def utc_now():
# utcnow returns a naive datetime, so we have to set the timezone manually <sigh>
return datetime.utcnow().replace(tzinfo=timezone.utc)
class Terminator:
pass
@drj42
drj42 / asyncio_producer_consumer.py
Created Jan 30, 2017 — forked from akrylysov/asyncio_producer_consumer.py
Python 3 asyncio basic producer / consumer example
View asyncio_producer_consumer.py
import asyncio
import random
q = asyncio.Queue()
async def producer(num):
while True:
await q.put(num + random.random())
await asyncio.sleep(random.random())
@drj42
drj42 / s3.py
Created Jan 27, 2017 — forked from stalkerg/s3.py
Async Tornado S3 uploader with AWS4 sign
View s3.py
import hashlib
import hmac
import mimetypes
import binascii
from calendar import timegm
from datetime import datetime
import time
from email.utils import formatdate
from urllib.parse import quote, urlparse
@drj42
drj42 / df_reduce_by_key.py
Created Mar 15, 2016
PySpark Helper Function - perform reduceByKey on a dataframe
View df_reduce_by_key.py
# Removes a lot of the boiler plate involved in converting a pyspark dataframe
# to and from an rdd, in order to do a reduceByKey operation.
#
# Lifted from:
# - http://codereview.stackexchange.com/questions/115082/generic-reduceby-or-groupby-aggregate-functionality-with-spark-dataframe
from pyspark.sql import Row
from pyspark.sql.functions import struct
from pyspark.sql import DataFrame
@drj42
drj42 / luigi_sftp.py
Last active Feb 8, 2016
sftp remote target for luigi
View luigi_sftp.py
"""sftp.py - PySftp connections wrapped up in a luigi.Target.
TODO: get rid of the redundant stuff, write some tests, contribute to luigi
upstream.
"""
# -*- coding: utf-8 -*-
import io
import os
import random
@drj42
drj42 / task_params.py
Created Nov 21, 2015
TaskParameter quirks
View task_params.py
import luigi
class Foo(luigi.Task):
message = 'Foo'
class RunOnceTask(luigi.Task):
my_task = luigi.TaskParameter()
@drj42
drj42 / gist:a1ff0e57e11e930291da
Last active Aug 29, 2015
Load csv's in spark
View gist:a1ff0e57e11e930291da
"""
Launch pyspark with the flag:
--packages com.databricks:spark-csv_2.10:1.0.3
"""
from pyspark.sql import SQLContext
from pyspark.sql.types import StructField, StringType, StructType
def create_schema(fields):
""" Create a dataframe schema from a list of field names.
You can’t perform that action at this time.