mkaranasou

#!/usr/bin/env python2.7

import random
import subprocess


class Node(object):
    def __init__(self, key, value):
        self.key = key
        self.value = value

mkaranasou

# -*- coding: utf-8 -*-
# Many thanks to http://stackoverflow.com/users/400617/davidism
# This code under "I don't care" license
# Take it, use it, learn from it, make it better.
# Start this from cmd or shell or whatever
# Go to favourite browser and type localhost:5000/admin
import sys
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.admin import Admin

mkaranasou

def union_uneven(df_base, df_new, default=None):
    """
    Union dfs with different columns
    :param: pyspark.DataFrame df_base: the dataframe to join to
    :param: pyspark.DataFrame df_new: the dataframe to be joined
    :return: the union of the two dataframes, having the missing columns filled with the default value
    :rtype: pyspark.DataFrame
    """
    base_columns = set(df_base.columns)
    df_new_columns = set(df_new.columns)

mkaranasou

json_col = 'json_col'

# either infer the features schema:
schema = self.spark.read.json(df.select(json_col).rdd.map(lambda x: x[0])).schema

# parse the features string into a map
df = df.withColumn(json_col, (F.from_json(F.col(json_col), schema)))

# access the feature columns by name
df.select(F.col(json_col)['some_key']).show()

mkaranasou

import EXIF from 'exif-js';

const hasBlobConstructor = typeof (Blob) !== 'undefined' && (function checkBlobConstructor() {
    try {
        return Boolean(new Blob());
    } catch (error) {
        return false;
    }
}());

mkaranasou

from pyspark import SparkConf
from pyspark.sql import SparkSession, functions as F

conf = SparkConf()
# optional but it would be good to set the amount of ram the driver can use to 
# a reasonable (regarding the size of the file we want to read) amount, so that we don't get an OOM exception
conf.set('spark.driver.memory', '6G')

# create a spark session - nothing can be done without this:
spark = SparkSession.builder \

mkaranasou

from pyspark import SparkConf
from pyspark.sql import SparkSession, functions as F

conf = SparkConf()
# optional but it would be good to set the amount of ram the driver can use to 
# a reasonable (regarding the size of the file we want to read) amount, so that we don't get an OOM exception
conf.set('spark.driver.memory', '6G')

spark = SparkSession.builder \
        .config(conf=conf) \

mkaranasou

>>> from pyspark.sql import SparkSession, functions as F
>>> from pyspark import SparkConf
>>> conf = SparkConf()

>>> spark = SparkSession.builder \
        .config(conf=conf) \
        .appName('Dataframe with Indexes') \
        .getOrCreate()

mkaranasou

import os
import re
import yaml


def parse_config(path=None, data=None, tag='!ENV'):
    """
    Load a yaml configuration file and resolve any environment variables
    The environment variables must have !ENV before them and be in this format
    to be parsed: ${VAR_NAME}.

mkaranasou

# To run this:
# export DB_PASS=very_secret_and_complex 
# python use_env_variables_in_config_example.py -c /path/to/yaml
# do stuff with conf, e.g. access the database password like this: conf['database']['DB_PASS']

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='My awesome script')
    parser.add_argument(
        "-c", "--conf", action="store", dest="conf_file",
        help="Path to config file"