lonly197

#!/bin/bash

# Set App Name
AppName="bumblebee-api"
# Set Version
Version="1.0.0"
# Set Jar Package File Name
SpringBoot="$AppName-$Version.jar"
# Set The Home Path
HomePath="/opt/bumblebee"

lonly197

#!/bin/sh

# store start date to a variable
imeron=`date`

echo "Import started: OK"
# set sql dump file
dumpfile="/home/lonly/big.sql"

ddl="set names utf8; "

lonly197

import com.google.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.glassfish.jersey.internal.util.Base64;

import java.io.*;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

/**

lonly197

import scala.collection.mutable.WrappedArray
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._

import breeze.linalg.{DenseVector => BDV, SparseVector => BSV, Vector => BV}
import org.apache.spark.ml.linalg.{DenseVector, Matrices, Matrix, SparseVector, Vector, Vectors}
import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
import org.apache.spark.sql.UDFRegistration
import streaming.common.UnicodeUtils

lonly197

import org.apache.spark.sql.DataFrame
import scala.annotation.tailrec

implicit class DataFrameOperations(df: DataFrame) {
  def dropDuplicateCols(rmvDF: DataFrame): DataFrame = {
    val cols = df.columns.groupBy(identity).mapValues(_.size).filter(_._2 > 1).keySet.toSeq

    @tailrec
    def deleteCol(df: DataFrame, cols: Seq[String]): DataFrame = {
      if (cols.size == 0) df else deleteCol(df.drop(rmvDF(cols.head)), cols.tail)

lonly197

import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.functions._

def concat(df1: DataFrame, df2: DataFrame): DataFrame = {
    
    val cols1 = df1.columns.toSet
    val cols2 = df2.columns.toSet
    val total = cols1 ++ cols2 // union

    def expr(myCols: Set[String], allCols: Set[String]) = {

lonly197

// Base64 encode
val text = "This is plaintext."
val bytesEncoded = java.util.Base64.getEncoder.encode(text.getBytes())

// Base64 decode
val textDecoded = new String(java.util.Base64.getDecoder.decode(bytesEncoded))
println(textDecoded)

lonly197

import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;

lonly197

import com.google.common.base.CaseFormat;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;

import java.beans.BeanInfo;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.lang.reflect.Method;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;

lonly197

import pandas as pd
import numpy as np

def find_correlation(df, thresh=0.9):
    """
    Given a numeric pd.DataFrame, this will find highly correlated features,
    and return a list of features to remove
    params:
    - df : pd.DataFrame
    - thresh : correlation threshold, will remove one of pairs of features with