flume.properties

#define sources, channels and sinks
agent_orange.sources = agent_orange_stock
agent_orange.channels = mem_spark mem_hdfs
agent_orange.sinks = hdfs_out spark_out
 
#configure sinks
agent_orange.sources.agent_orange_stock.type = avro
agent_orange.sources.agent_orange_stock.bind = 0.0.0.0
agent_orange.sources.agent_orange_stock.port = 4141
 
#configure channel
agent_orange.channels.mem_hdfs.type = memory
agent_orange.channels.mem_hdfs.capacity = 1000
 
agent_orange.channels.mem_spark.type = memory
agent_orange.channels.mem_spark.capacity = 100
 
#configure sinks
agent_orange.sinks.hdfs_out.type = hdfs
agent_orange.sinks.hdfs_out.hdfs.path = /flume/symbol=%{symbol}
agent_orange.sinks.hdfs_out.hdfs.fileType = DataStream
agent_orange.sinks.hdfs_out.hdfs.rollSize = 0
agent_orange.sinks.hdfs_out.hdfs.rollCount = 100
#collector.sinks.hdfs_out.hdfs.rollInterval = 5
agent_orange.sinks.hdfs_out.hdfs.batchSize = 100
agent_orange.sinks.hdfs_out.serializer = org.apache.flume.serialization.AvroEventSerializer$Builder
#collector.sinks.hdfs_out.serializer.compressionCodec = snappy
 
agent_orange.sinks.spark_out.type = avro
agent_orange.sinks.spark_out.hostname = localhost
agent_orange.sinks.spark_out.port = 9999
agent_orange.sinks.spark_out.batch-size = 1
 
 
#join source and sinks to channels
agent_orange.sources.agent_orange_stock.channels = mem_spark
agent_orange.sinks.hdfs_out.channel = mem_hdfs
agent_orange.sinks.spark_out.channel = mem_spark