y2k-shubham

Find regex: `(?<!\w)(insert|is|into|with|as|select|from|where|between|and|or|distinct|case|when|then|else|end|on|null|not|inner|outer|full|join|left|right|group|by|order|limit|in)(?!\w)`
Replace regex: `\U$1\E`

y2k-shubham

#!/bin/bash
for user in $( cat users.txt ); do
    # create user
    sudo useradd --create-home $user
    # generate random password
    PASSWD="$(pwgen -cnsv -N 1 32)"
    # set password
    echo $user:$PASSWD | chpasswd
    # print message
    echo "Created user $user with password $PASSWD"

y2k-shubham

-- DO NOT use this; see comment below
CREATE EXTERNAL TABLE `db_name.table_name`(
  `column_1` string,
  `column_2` string)
ROW FORMAT SERDE
  'org.apache.hadoop.hive.serde2.OpenCSVSerde'
WITH SERDEPROPERTIES (
  'escapeChar'='\\',
  'quoteChar'='"',
  'separatorChar'=',')

y2k-shubham

from typing import List, Optional, Dict, Any

from airflow.contrib.hooks.ssh_hook import SSHHook
from airflow.contrib.operators.ssh_operator import SSHOperator
from airflow.exceptions import AirflowException
from airflow.utils import apply_defaults

import ssh_utils

y2k-shubham

#!/bin/zsh
# pyenv install for CentOS 6.5 x86_64 

yum install -y  gcc gcc-c++ make git patch openssl-devel zlib-devel readline-devel sqlite-devel bzip2-devel

git clone git://github.com/yyuu/pyenv.git ~/.pyenv

export PATH="$HOME/.pyenv/bin:$PATH"
eval "$(pyenv init -)"

y2k-shubham

from airflow.settings import Session
from airflow.utils.db import provide_session
from airflow.models import Pool

# hive_pool is just an example, you might want to create some other types of pools such as for MySQL
@provide_session
def create_hive_pool(session: Optional[Session] = None) -> None:
    pool = Pool(pool=pool_templates['hive_name'],
        		slots=1,
        		description=pool_templates['hive_description'])

y2k-shubham

2018-12-04T12:29:54.433+0530	WARN	hive-hive-63	com.facebook.presto.hive.util.ResumableTasks	ResumableTask completed exceptionally
java.lang.NoClassDefFoundError: Could not initialize class com.amazon.ws.emr.hadoop.fs.util.EmrFsUtils
	at com.amazon.ws.emr.hadoop.fs.s3n.S3Credentials.initialize(S3Credentials.java:45)
	at com.amazon.ws.emr.hadoop.fs.HadoopConfigurationAWSCredentialsProvider.<init>(HadoopConfigurationAWSCredentialsProvider.java:26)
	at com.amazon.ws.emr.hadoop.fs.guice.DefaultAWSCredentialsProviderFactory.getAwsCredentialsProviderChain(DefaultAWSCredentialsProviderFactory.java:44)
	at com.amazon.ws.emr.hadoop.fs.guice.DefaultAWSCredentialsProviderFactory.getAwsCredentialsProvider(DefaultAWSCredentialsProviderFactory.java:28)
	at com.amazon.ws.emr.hadoop.fs.guice.EmrFSProdModule.getAwsCredentialsProvider(EmrFSProdModule.java:65)
	at com.amazon.ws.emr.hadoop.fs.guice.EmrFSProdModule.createS3Configuration(EmrFSProdModule.java:81)
	at com.amazon.ws.emr.hadoop.fs.guice.EmrFSProdModule.createAmazonS3Li

y2k-shubham

#/bin/sh

# create directories
mkdir -p /{main,test}/{resources,scala/com/zomato}
mkdir -p {nonsvn,project,target}

# create build.sbt file
echo '
	name := "ProjectName"
	version := "1.0"

y2k-shubham

import java.sql.Connection

import com.typesafe.scalalogging.Logger
import com.company.utils.jdbc.MySQLConfig
import scalikejdbc.{ConnectionPool, ConnectionPoolSettings, DB, using}

import scala.util.control.NonFatal

object ConnectionPoolManager {

y2k-shubham

Reference files for integration test of a Presto plugin containing hashing-related UDFs