Kafka+Spark.sh

#!/bin/bash
# Spark
mkdir ~/spark && cd ~/spark && \
    wget http://apache.claz.org/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz -O spark.tgz && \
    tar zxvf spark.tgz --strip 1 && \
    rm spark.tgz
    
pip install pyspark
pip install kafka-python

# kafka
# https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-18-04
# Since Kafka can handle requests over a network, you should create a dedicated user for it.
# This minimizes damage to your Ubuntu machine should the Kafka server be compromised
## Creating a User for Kafka
sudo useradd kafka -m
sudo passwd kafka
sudo adduser kafka sudo
su -l kafka

## Downloading and Extracting the Kafka Binaries
mkdir ~/kafka && cd ~/kafka && \
    wget "http://www-eu.apache.org/dist/kafka/1.1.0/kafka_2.12-1.1.0.tgz" -O kafka.tgz && \
    tar zxvf kafka.tgz --strip 1 && \
    rm kafka.tgz

## Configuring the Kafka Server
# Kafka's default behavior will not allow us to delete a topic, the category, group, or feed name to which messages can be published
# ```echo "delete.topic.enable=true" >> ~/kafka/config/server.properties``` in case this append to the end of line but not a new line
echo -e "\ndelete.topic.enable=true" >> ~/kafka/config/server.properties

## Creating Systemd Unit Files and Starting the Kafka Server
# https://stackoverflow.com/questions/10134901/why-sudo-cat-gives-a-permission-denied-but-sudo-vim-works-fine
### Create the unit file for zookeeper
sudo bash -c 'cat>>/etc/systemd/system/zookeeper.service' <<EOF
[Unit]
Requires=network.target remote-fs.target
After=network.target remote-fs.target
[Service]
Type=simple
User=kafka
ExecStart=/home/kafka/kafka/bin/zookeeper-server-start.sh /home/kafka/kafka/config/zookeeper.properties
ExecStop=/home/kafka/kafka/bin/zookeeper-server-stop.sh
Restart=on-abnormal
[Install]
WantedBy=multi-user.target
EOF

### create the systemd service file for kafka:
sudo tee -a /etc/systemd/system/kafka.service >/dev/null <<EOF
[Unit]
Requires=zookeeper.service
After=zookeeper.service
[Service]
Type=simple
User=kafka
ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties > /home/kafka/kafka/kafka.log 2>&1'
ExecStop=/home/kafka/kafka/bin/kafka-server-stop.sh
Restart=on-abnormal
[Install]
WantedBy=multi-user.target
EOF

### start Kafka
sudo systemctl start kafka

# To ensure that the server has started successfully, check the journal logs for the kafka unit:
# user kafka cannot see the journal logs, since it is not in groups 'adm', 'systemd-journal'
# Hint: You are currently not seeing messages from other users and the system.
#       Users in groups 'adm', 'systemd-journal' can see all messages.
#       Pass -q to turn off this notice.
### so run this on your own user
journalctl -u kafka

### To enable kafka on server boot
sudo systemctl enable kafka

## Testing the Installation
### create a topic named test
~/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
bin/kafka-topics.sh --list --zookeeper localhost:2181
 
### Run the producer to send some messages
echo "Hello, World" | ~/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test > /dev/null

### Start a consumer in another terminal
~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning

### Remove test topic
~/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --delete --topic test

# Restricting the Kafka User

# Remove the kafka user from the sudo group
sudo deluser kafka sudo

# lock the kafka user's password
sudo passwd kafka -l

# At this point, only root or a sudo user can log in as kafka by typing in the following command:
# sudo su - kafka

# unlock
# sudo passwd kafka -u