flume-logger-hdfs.conf

# flume-logger-hdfs.conf: Read data from logs and write it to both logger and hdfs
# flume command to start the agent - flume-ng agent --name a1 --conf /home/dgadiraju/flume_example/example.conf --conf-file example.conf

# Name the components on this agent
a1.sources = logsource
a1.sinks = loggersink hdfssink
a1.channels = loggerchannel hdfschannel

# Describe/configure the source
a1.sources.logsource.type = exec
a1.sources.logsource.command = tail -F /opt/gen_logs/logs/access.log

# Describe the sink
a1.sinks.loggersink.type = logger

# Use a channel which buffers events in memory
a1.channels.loggerchannel.type = memory
a1.channels.loggerchannel.capacity = 1000
a1.channels.loggerchannel.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.logsource.channels = loggerchannel hdfschannel
a1.sinks.loggersink.channel = loggerchannel

#Describe the sink
a1.sinks.hdfssink.type = hdfs
a1.sinks.hdfssink.hdfs.path = hdfs://nn01.itversity.com:8020/user/dgadiraju/flume_example_%Y-%m-%d
a1.sinks.hdfssink.hdfs.fileType = DataStream
a1.sinks.hdfssink.hdfs.rollInterval = 120
a1.sinks.hdfssink.hdfs.rollSize = 10485760
a1.sinks.hdfssink.hdfs.rollCount = 30
a1.sinks.hdfssink.hdfs.filePrefix = retail
a1.sinks.hdfssink.hdfs.fileSuffix = .txt
a1.sinks.hdfssink.hdfs.inUseSuffix = .tmp
a1.sinks.hdfssink.hdfs.useLocalTimeStamp = true

#Use a channel which buffers events in file for HDFS sink
a1.channels.hdfschannel.type = file
a1.channels.hdfschannel.capacity = 1000
a1.channels.hdfschannel.transactionCapacity = 100
a1.channels.hdfschannel.checkpointInterval = 300

a1.sinks.hdfssink.channel = hdfschannel

This is very useful Thank you so much for contribution