nalingarg2

#
# Hadoop:: flumeHive
# Recipe:: Flume and Hive
#
# Copyright (C) 2015 Cloudwick labs
# Contact :: nalin.garg@cloudwick.com
# All rights reserved - Do Not Redistribute
#

#install sbt

nalingarg2

# remote mestaore, sql , hive-client

# hive-client
yum install hive

#hive metastore
yum install hive-metstore

#change hive configuration
<?xml version="1.0"?>

nalingarg2

# reference: http://transcriptome.ens.fr/eoulsan/hadoopmode.html

core-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

nalingarg2

[mysqld]

#transaction-isolation = READ-COMMITTED

datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
user=mysql
# Disabling symbolic-links is recommended to prevent assorted security$
symbolic-links=0

nalingarg2

#https://docs.puppetlabs.com/learning/agent_master_basic.html

┌───────────────┬───────────────────┐
│ Pre-2.6       │ Post-2.6          │
├───────────────┼───────────────────┤
│ puppetmasterd │ puppet master     │
│ puppetd       │ puppet agent      │
│ puppet        │ puppet apply      │
│ puppetca      │ puppet cert       │
│ ralsh         │ puppet resource   │

nalingarg2

To create the MySQL databases for Cloudera Manager:

Log into MySQL as the root user:
$ mysql -u root -p
Enter password:
Create a database for the Activity Monitor. The database name, user name, and password can be anything you want. For example:
mysql> create database amon DEFAULT CHARACTER SET utf8;
Query OK, 1 row affected (0.00 sec)

mysql> grant all on amon.* TO 'amon'@'%' IDENTIFIED BY 'amon_password';

nalingarg2

http://blog.cloudera.com/blog/2013/03/how-to-create-a-cdh-cluster-on-amazon-ec2-via-cloudera-manager/

nalingarg2

#
# Hadoop:: KafkaSpark
# Recipe:: Kafka and Spark
#
# Copyright (C) 2015 Cloudwick labs
# Contact :: nalin.garg@cloudwick.com
# All rights reserved - Do Not Redistribute
#

#One machine is required as below mentioned steps are for POC purpose only.

nalingarg2

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>df</groupId>
    <artifactId>df</artifactId>
    <version>1.0-SNAPSHOT</version>

nalingarg2

git init .
git add .
git commit -am "my first commit"

git status 

git remote add kinesis https://github.com/nalingarg2/Grad101Infrastructure.git

git remote -v