LeiG

#!/bin/zsh

# gcloud-init initializes a Google dataproc cluster
# arguments:
#   $1 jar
#   $2 env
function gcloud-init () {
    gcloud dataproc clusters create dp-$USER-$1-test \
    --image-version 1.1 \
    --num-workers 20 \

LeiG

;; -*- mode: emacs-lisp -*-
;; This file is loaded by Spacemacs at startup.
;; It must be stored in your home directory.

(defun dotspacemacs/layers ()
  "Configuration Layers declaration.
You should not put any user code in this function besides modifying the variable
values."
  (setq-default
   ;; Base distribution to use. This is a layer contained in the directory

LeiG

import yaml
import pyodbc
import impala.dbapi
import impala.util

with open('config.yaml', 'r') as f:
  cfg = yaml.load(f)
  
# METHOD 1: using pyodbc to establish connection
connString = 'Driver=%s;Host=%s;Port=%d;Database=default; \

LeiG

import rpy2.robjects as robjects
import pandas.rpy.common as com
import pandas as pd

## load .RData and converts to pd.DataFrame
robj = robjects.r.load('test.RData')
# iterate over datasets the file
for sets in robj:
    myRData = com.load_data(sets)
    # convert to DataFrame