public
Created

Flume Configuration files

  • Download Gist
flume-site-agent.xml
XML
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
<!-- a1 (agent) -->
<configuration>
<property>
<name>flume.master.servers</name>
<value>$master_IP</value>
<description>This is the address for the config servers status server (http)</description>
</property>
 
<property>
<name>flume.collector.event.host</name>
<value>$collector_IP</value>
<description>This is the host name of the default "remote" collector.</description>
</property>
 
<property>
<name>flume.collector.port</name>
<value>35853</value>
<description>This default tcp port that the collector listens to in order to receive events it is collecting.</description>
</property>
 
<property>
<name>flume.agent.logdir</name>
<value>/mnt/flume-${user.name}/agent</value>
<description> This is the directory that write-ahead logging data
or disk-failover data is collected from applications gets
written to. The agent watches this directory.
</description>
</property>
</configuration>
flume-site-collector.xml
XML
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
<!-- c1 (collector) /etc/flume/conf/flume-site.xml -->
<configuration>
<property>
<name>flume.master.servers</name>
<value>$master</value>
<description>This is the address for the config servers status server (http)</description>
</property>
 
<property>
<name>flume.collector.event.host</name>
<value>localhost</value>
<description>This is the host name of the default "remote" collector.</description>
</property>
 
<property>
<name>flume.collector.port</name>
<value>35853</value>
<description>This default tcp port that the collector listens to in order to receive events it is collecting.</description>
</property>
 
<property>
<name>fs.default.name</name>
<value>s3n://$account:$secret@$bucket</value>
</property>
 
<property>
<name>fs.s3n.impl</name>
<value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value>
</property>
<property>
<name>fs.s3.awsAccessKeyId</name>
<value>$account</value>
</property>
<property>
<name>fs.s3.awsSecretAccessKey</name>
<value>$secret</value>
</property>
 
<property>
<name>fs.s3n.awsAccessKeyId</name>
<value>$account</value>
</property>
<property>
<name>fs.s3n.awsSecretAccessKey</name>
<value>$secret</value>
</property>
 
<property>
<name>flume.agent.logdir</name>
<value>/mnt/flume-${user.name}/agent</value>
<description> This is the directory that write-ahead logging data
or disk-failover data is collected from applications gets
written to. The agent watches this directory.
</description>
</property>
<property>
<name>flume.collector.dfs.dir</name>
<value>file:///mnt/flume-${user.name}/collected</value>
<description>This is a dfs directory that is the the final resting
place for logs to be stored in. This defaults to a local dir in
/tmp but can be hadoop URI path that such as hdfs://namenode/path/
</description>
</property>
 
<property>
<name>flume.collector.dfs.compress.gzip</name>
<value>true</value>
<description>Writes compressed output in gzip format to dfs. value is
boolean type, i.e. true/false</description>
</property>
 
<property>
<name>flume.collector.roll.millis</name>
<value>60000</value>
<description>The time (in milliseconds)
between when hdfs files are closed and a new file is opened
(rolled).
</description>
</property>
</configuration>

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.