prakshalj0512

# Install Python 3.7
yum install -y python37
yum install -y python3-pip
yum install -y zip

# Create virutal environment
python3.7 -m pip install --upgrade pip
python3.7 -m pip install virtualenv
python3.7 -m venv snowflake-venv
source snowflake-venv/bin/activate

prakshalj0512

# UPDATE THE FOLLOWING VARIABLES IN THE CODE

# SNOWFLAKE_ACCOUNT: the account number of Snowflake (can be found in the 
# SNOWFLAKE_USER: Snowflake username
# SNOWFLAKE_PASSWORD: Snowflake password
# SNOWFLAKE_QUERY: the Snowflake query to run 
# S3_BUCKET: name of S3 bucket to store results
# S3_OUTPUT_KEY: name of the file to store with the results

import boto3

prakshalj0512

"""

AUTH_ROLE_ADMIN: the role of the bind user (should be Admin)
AUTH_USER_REGISTRATION: boolean for automatically creating users on first log-in

AUTH_USER_REGISTRATION_ROLE: the role which first-time users logging in will be assigned
                             Possible Values: Admin, Viewer, User, Op, Public 

AUTH_LDAP_SERVER: the LDAP server URI
AUTH_LDAP_SEARCH: update with the LDAP path under which you’d like the users to have access to Airflow (e.g. : 'dc=example,dc=com')

prakshalj0512

"""

AUTH_ROLE_ADMIN: the role of the bind user (should be Admin)
AUTH_USER_REGISTRATION: boolean for automatically creating users on first log-in

AUTH_USER_REGISTRATION_ROLE: the role which first-time users logging in will be assigned
                             Possible Values: Admin, Viewer, User, Op, Public 

AUTH_LDAP_SERVER: the LDAP server URI
AUTH_LDAP_SEARCH: update with the LDAP path under which you’d like the users to have access to Airflow. Example: dc=example,dc=com.

prakshalj0512

/* UPDATE THE FOLLOWING VARIABLES IN THE CODE

aws-account-no: the account number of the AWS account associated with the lambda function
lambda-function-name: the name of the lambda function to deploy
lambda-function-region: the region in which the Lambda function will be deployed, such as us-west-2.
s3-bucket: name of the S3 bucket to store the code created earlier

*/

prakshalj0512

/* UPDATE THE FOLLOWING VARIABLES IN THE CODE

aws-act-no: the account number associated with the Dynamodb table (such as 657065845823
cf-stack-region: the region of the CloudFormation stack (such as us-west-2)
cf-stack-name: name of the CloudFormation responsible for deploying the lambda function
lambda-function-region: the region of the lambda function (such as us-west-2)
s3-bucket-name: name of the S3 bucket where the Lambda function artifacts will be stored

*/

prakshalj0512

echo "S3 Bucket: $S3_BUCKET"
echo "Lambda Function Name: $LAMBDA_FUNCTION_NAME"
echo "Lambda Deployment Preference: $LAMBDA_DEPLOYMENT_PREFERENCE"
echo "GitHub Branch: $BRANCH"

if [ "$BRANCH" == "master" ]; then
    BRANCH="prod"
fi

FUNCTION_EXISTS=$(aws lambda wait function-exists --function-name ${LAMBDA_FUNCTION_NAME}-${BRANCH})

prakshalj0512

version: 0.2

phases:
  build:
    commands:
      - . scripts/deploy.sh
      - sam package --template-file template.yaml --s3-bucket ${S3_BUCKET} --output-template-file packaged.yaml
artifacts:
  files:
    - "packaged.yaml"

prakshalj0512

/* UPDATE THE FOLLOWING VARIABLES IN THE CODE

s3-bucket-name: name of the S3 bucket
dynamodb-region: the region of the Dynamodb table (such as us-west-2)
dynamodb-act-no: the account number associated with the Dynamodb table (such as 657065845823
dynamodb-table-name: name of the Dynamodb table

*/

{

prakshalj0512

#!/bin/bash

NIFI_HOST="localhost:8080"
HADOOP_CORE_SITE_PATH="/etc/hadoop/conf/core-site.xml"
HADOOP_HDFS_SITE_PATH="/etc/hadoop/conf/hdfs-site.xml"
TEMP_HDFS_DIRECTORY="/tmp/nifiTest"

# Retrieve Root Process ID
mainProcessID=$(curl -s http://{$NIFI_HOST}/nifi-api/flow/process-groups/root/status?recursive=true |
    python -c "import sys, json; print (json.load(sys.stdin)['processGroupStatus']['id'])")