Skip to content

Instantly share code, notes, and snippets.

View dyno's full-sized avatar
🏠
Working from home

Dyno Fu dyno

🏠
Working from home
View GitHub Profile
# Uncrustify 0.59
#
# General options
#
# The type of line endings
newlines = auto # auto/lf/crlf/cr
# The original size of tabs in the input
@dyno
dyno / bayarea.ipynb
Created March 29, 2016 20:08
The Bay Area Fact with Geodata
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
@dyno
dyno / pbopen.py
Created January 17, 2017 17:55
drop-in replacement of file object line iterator with tqdm progress bar
#!/usr/bin/env python
from contextlib import contextmanager
from os.path import getsize, basename
from tqdm import tqdm
@contextmanager
def pbopen(filename):
total = getsize(filename)
@dyno
dyno / sparkSessionOnEMR.scala
Last active October 3, 2018 22:26
Working with SparkSession with Ammonite .on AWS EMR
// =============================================================================
// emr.sc -- import it in predef.sc
import ammonite.ops._
// add emr jars and config dir to interp class path
//
def loadEmrJars() = {
// /etc/zeppelin/conf/zeppelin-env.sh
val libDirs = List( //
"/usr/lib/hadoop-lzo/lib",
"/usr/lib/spark/jars",
import re
from xml.etree import ElementTree as ET
import requests
EMR_RELEASE_NOTES_RSS_URL = "https://docs.aws.amazon.com/emr/latest/ReleaseGuide/amazon-emr-release-notes.rss"
def get_latest_emr_release():
"Get latest release version from EMR release note"
@dyno
dyno / find_jar_conflict.py
Created October 17, 2018 04:26
Print out java class path to jars that contains it
spark_jars = "file://x.jar,file://y.jar"
jars = spark_jars.split(",")
from zipfile import ZipFile
from pathlib import Path
from collections import defaultdict
from pprint import pprint
libs = defaultdict(set)
for jar in sorted(jars):
@dyno
dyno / bigdata-env.sh
Last active October 22, 2018 04:16
local big data env
#!/usr/bin/env bash
# Setup environment variables required to execute hadoop/spark/zeppelin on Mac
#XXX: Supposed to be sourced in, don't complain about the indent.
if [ "$(uname)" = "Darwin" ]; then
# yarn and hadoop cannot be both installed.
brew unlink yarn &>/dev/null || true
@dyno
dyno / publish_pull_venv_lockfile.gradle
Last active January 18, 2019 03:32
publish Pipfile.lock Poetry.lock as artifacts
logging.captureStandardOutput LogLevel.INFO
def nexusUsername = "builder"
def nexusPassword = 'password'
repositories {
maven {
name = "example-releases"
url 'https://nexus.example.com/nexus/content/repositories/releases'
credentials {
from random import randint, seed
a = "what"
def f1():
print(a)
a = "ever"
print(a)
@dyno
dyno / zanmei.py
Last active February 4, 2019 06:18
Download "Hymns for God's People" from Zanmei
#!/usr/bin/env python3
# vim: set fileencoding=utf-8 :
import re
import shutil
from pathlib import Path
from typing import Dict
import attr
import requests