mndrake

#!/usr/bin/env python
# coding: utf-8

# In[11]:


# dependencies
import pytesseract
from bs4 import BeautifulSoup
from PIL import Image

mndrake

import os
from io import BytesIO
import pyarrow as pa
import pyarrow.parquet as pq


kilobytes = 1024
megabytes = kilobytes * 1000
chunksize = int(10 * megabytes)

mndrake

from urllib.parse import urlparse
import re
import dataiku
import pandas as pd

PROJECT_ID = 'CUSTOMERSEGMENTATION'
ANALYSIS_ID = 'UjW24hJ1'
ML_TASK_ID = 'LsiobCLw'
MODEL_ID = 'A-CUSTOMERSEGMENTATION-UjW24hJ1-LsiobCLw-s1-pp1-m1'

mndrake

# originally from: http://stackoverflow.com/questions/42528400/plot-brushing-or-accessing-drawn-shape-geometry-for-spatial-subsets-in-shiny-lea
# uses https://github.com/bhaskarvk/leaflet.extras

library(shiny)
library(leaflet)
library(leaflet.extras)
library(sp)

cities <- structure(list(AccentCity = c("Saint Petersburg", "Harare", "Qingdao", 
                                        "Addis Abeba", "Xian", "Anshan", "Rongcheng", "Kinshasa", "New York", 

mndrake

from io import StringIO
import logging

class StreamingLog(object):
    def __init__(self, logger_name, level=logging.INFO):
        self.stream = StringIO()
        self.handler = logging.StreamHandler(self.stream)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        self.handler.setFormatter(formatter)
        self.log = logging.getLogger(logger_name)

mndrake

library(miniCRAN)
library(remotes)

# CRAN mirror to use (recommend checkpoint date for installed version of R https://mran.microsoft.com/timemachine)
cran_repo <- c(CRAN = "https://cran.microsoft.com/snapshot/2018-11-30")

# local path to create miniCRAN repo
miniCRAN_dir <- "/data/dataiku/miniCRAN"

mndrake

#!/usr/bin/python
# Filename: sas_export.py
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 06 18:40:09 2015
@author: David Carlson
modified version for sas7bdat 2.0.1 of Charlie Huang version at:
http://www.sasanalysis.com/2014/08/python-extension-functions-to-translate.html
"""

mndrake

#!python
import functools
import json
import os
import requests
import urllib 
import uuid
import configparser

# TODO: CURRENTLY ONLY WORKS FOR AWS, NEED TO ADD ADDITIONAL PARSING FOR AZURE

mndrake

from IPython.core.magic import line_magic, line_cell_magic, Magics, magics_class
from pyspark.sql import SparkSession
from pyspark.dbutils import DBUtils

spark = SparkSession.builder.getOrCreate()
sc = spark.sparkContext
dbutils = DBUtils(sc)

@magics_class
class DatabricksConnectMagics(Magics):

mndrake

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

namespace Utility