NIFI + active mq+ kafka + data flow+ Vagrant

Basic Nifi Set up for JMS data flow

- Run the below Vagrant to bring up the necessary nifi, active mq, zookeper and kafka
- Create a dummy que name "test" in activemq
- Create a dummy topic "test" in kafka
- start nifi
- navigate to localhost:8080/nifi/
- From Dashboard configure the source and sink processor defination

- https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#clustering
- http://ingest.tips/2014/12/22/getting-started-with-apache-nifi/

Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :
$script = <<SCRIPT
echo Installing dependencies...
yum -y update
yum -y install unzip wget
sudo yum install -y epel-release
sudo yum install -y unzip
sudo yum install -y curl
sudo yum install -y python-pip
sudo yum install -y java-1.8.0-openjdk-devel
pip install --upgrade pip
echo "Installing Docker ..."
yum check-update
curl -fsSL https://get.docker.com/ | sh
sudo systemctl enable docker.service
pip install docker-compose --force --upgrade
cd /dev/workspace
echo "Downloading NIFI"
wget http://mirror.fibergrid.in/apache/nifi/1.1.2/nifi-1.1.2-bin.tar.gz
tar xvf nifi-1.1.2-bin.tar.gz 

echo "DownLoading ActiveMq"
wget https://www.apache.org/dist/activemq/5.14.4/apache-activemq-5.14.4-bin.tar.gz
tar xvf apache-activemq-5.14.4-bin.tar.gz 

sudo adduser --system activemq
sudo chown -R activemq: /dev/workspace/apache-activemq-5.14.4/

echo "Downloading Kafka"
wget http://www-us.apache.org/dist/kafka/0.9.0.1/kafka_2.11-0.9.0.1.tgz
tar xvf kafka_2.11-0.9.0.1.tgz 


echo "Commands to follow"

#Starting and stopping nifi
#NIFI_HOME/bin/nifi.sh start/stop URL:localhost:8188/nifi

#Active Mq make the below change in vi /dev/workspace/apache-activemq-5.14.4/conf/activemq.xml 
#<managementContext>
#            <managementContext createConnector="true"/>
#        </managementContext>

# sudo /dev/workspace/apache-activemq-5.14.4/bin/activemq start &


#Kafka
#KAFKA_HOME/bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
#bin/kafka-server-start.sh config/server.properties starting the kafka
#CReate Topic: bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
#bin/kafka-topics.sh --list --zookeeper localhost:2181  list topics
#bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test :inline publishing
#bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning   :forconsuming

SCRIPT

# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2"

Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

  config.vm.provision "shell", inline: "echo Hello"
  # The most common configuration options are documented and commented below.
  # For a complete reference, please see the online documentation at
  # https://docs.vagrantup.com.
  # 
  # 

  config.vm.box = "geerlingguy/centos7"
  config.vm.provision "shell", inline: $script
  config.vm.synced_folder "../../nifi_demo/", "/dev/workspace"
  config.vm.synced_folder '.', '/vagrant', disabled: true

  config.vm.define "nifi_local", autostart: true, primary:true do |nifi_local|

    nifi_local.vm.network "forwarded_port", guest: 8080, host: 8188
    nifi_local.vm.network "forwarded_port", guest: 8161, host: 8162
    nifi_local.vm.provider "virtualbox" do |vb|
      # Display the VirtualBox GUI when booting the machine
      vb.gui = false
      # Customize the amount of memory on the VM:
      vb.memory = "5120"
      vb.name = "Nifi Demo"
      vb.cpus = 4
    end
    #
    # Enable provisioning with a shell script. Additional provisioners such as
    # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
    # documentation for more information about their specific syntax and use.
    nifi_local.vm.provision "shell", inline: <<-SHELL
         echo "Hello Nifi Demo developer"
    SHELL
    #

  end
end