jayhuang75

yum update
yum install scl-utils
yum install centos-release-scl-rh
yum install python27
python -V
scl enable python27 bash

jayhuang75

cd /usr/hdp/current/kafka-broker/

## create topic
./kafka-topics --create --zookeeper sandbox.hortonworks.com:2181 --replication-factor 1 --partitions 1 --topic TOPIC_NAME

## list topics
./kafka-topics.sh --list --zookeeper sandbox.hortonworks.com:2181

## pub
./kafka-console-producer.sh --broker-list sandbox.hortonworks.com:6667 --topic TOPIC_NAME

jayhuang75

[[bin]] # Bin to run the gRPC client
name = "rust-grpc-kafka-pipeline-client"
path = "src/bin/client.rs"

jayhuang75

    // Init logging
    env_logger::init();

    let channel = tonic::transport::Channel::from_static("http://[::0]:50051")
        .connect()
        .await?;
    let mut client = PlayerServiceClient::new(channel);

    let mut sent_data = Vec::new();

jayhuang75

    // Load Env Config
    let config = config::Config::new().unwrap_or_else(|err| {
        panic!("Load Environment variable failed: {}", err);
    });

    // Init the GRPC server
    let addr = "[::0]:50051".parse().unwrap();
    let mut player_serivce = MyPlayerService::default();
    player_serivce.config = config;
    info!("[server.rs] PlayerServer listening on {}", addr);

jayhuang75

// MapReduce job example
hadoop jar hadoop-mapreduce-example-2.7.1-sources.jar org.apache.hadoop.examples.WordCount input output

// Hive job example
hive -f example_inert.hql

jayhuang75

	File System Counters
		FILE: Number of bytes read=52
		FILE: Number of bytes written=229301
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
		HDFS: Number of bytes read=0
		HDFS: Number of bytes written=25
		HDFS: Number of read operations=5
		HDFS: Number of large read operations=0

jayhuang75

// copy from hdfs to local container
hdfs dfs -copyToLocal <log-id>

// copy from local container to local machine
docker cp <log-id> <your-path-of-rust-project>

jayhuang75

.ONESHELL:
RELEASE_BIN=rust-hadoop-batch-control
RELEASE_BIN_PATH_LINUX=target/release
RELEASE_BIN_PATH_DARWIN=target/x86_64-unknown-linux-musl/release
HADOOP_NAMENODE=namenode
DB_LOCATION=db
DB_NAME=apps.db

.PHONY: darwin
darwin: 

jayhuang75

root@c1f89f553e01:/# DATABASE_URL=db/apps.db ./rust-hadoop-batch-control 
[main.rs] total apps in db : 13
[main.rs] current log total count: 13
[main.rs] time elapsed in run() is: 1.879390475s
[main.rs] total apps in db : 13
[main.rs] current log total count: 13
[main.rs] time elapsed in run() is: 1.860675949s
[main.rs] total apps in db : 13
[main.rs] current log total count: 13
[main.rs] time elapsed in run() is: 1.818562553s