This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/zsh | |
# gcloud-init initializes a Google dataproc cluster | |
# arguments: | |
# $1 jar | |
# $2 env | |
function gcloud-init () { | |
gcloud dataproc clusters create dp-$USER-$1-test \ | |
--image-version 1.1 \ | |
--num-workers 20 \ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
;; -*- mode: emacs-lisp -*- | |
;; This file is loaded by Spacemacs at startup. | |
;; It must be stored in your home directory. | |
(defun dotspacemacs/layers () | |
"Configuration Layers declaration. | |
You should not put any user code in this function besides modifying the variable | |
values." | |
(setq-default | |
;; Base distribution to use. This is a layer contained in the directory |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import yaml | |
import pyodbc | |
import impala.dbapi | |
import impala.util | |
with open('config.yaml', 'r') as f: | |
cfg = yaml.load(f) | |
# METHOD 1: using pyodbc to establish connection | |
connString = 'Driver=%s;Host=%s;Port=%d;Database=default; \ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import rpy2.robjects as robjects | |
import pandas.rpy.common as com | |
import pandas as pd | |
## load .RData and converts to pd.DataFrame | |
robj = robjects.r.load('test.RData') | |
# iterate over datasets the file | |
for sets in robj: | |
myRData = com.load_data(sets) | |
# convert to DataFrame |