Last active
June 27, 2018 06:59
-
-
Save m0zes/e6eb5ca39153989f7a37947a469e0b98 to your computer and use it in GitHub Desktop.
hadoop configuration with ceph
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0"?> | |
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> | |
<configuration> | |
<property> | |
<name>hadoop.proxyuser.mapred.groups</name> | |
<value>*</value> | |
</property> | |
<property> | |
<name>hadoop.proxyuser.mapred.hosts</name> | |
<value>*</value> | |
</property> | |
<!-- | |
Set-up the default filesystems | |
--> | |
<property> | |
<name>fs.default.name</name> | |
<value>ceph://hobbit01.beocat.ksu.edu:6789/</value> | |
<final /> | |
</property> | |
<property> | |
<name>fs.defaultFS</name> | |
<value>ceph://hobbit01.beocat.ksu.edu:6789/</value> | |
<final /> | |
</property> | |
<!-- | |
Configure non-default ceph options | |
--> | |
<property> | |
<name>ceph.auth.id</name> | |
<value>hadoop</value> | |
<final /> | |
</property> | |
<property> | |
<name>ceph.auth.keyring</name> | |
<value>/etc/ceph/ceph.client.hadoop.keyring</value> | |
<final /> | |
</property> | |
<property> | |
<name>ceph.data.pools</name> | |
<value>hadoop</value> | |
<final /> | |
</property> | |
<property> | |
<name>ceph.root.dir</name> | |
<value>/hadoop</value> | |
<final /> | |
</property> | |
<property> | |
<name>fs.AbstractFileSystem.ceph.impl</name> | |
<value>org.apache.hadoop.fs.ceph.CephFs</value> | |
<final /> | |
</property> | |
<property> | |
<name>fs.ceph.impl</name> | |
<value>org.apache.hadoop.fs.ceph.CephFileSystem</value> | |
<final /> | |
</property> | |
<property> | |
<name>ceph.replication</name> | |
<value>2</value> | |
<final /> | |
</property> | |
</configuration> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0"?> | |
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> | |
<configuration> | |
<property> | |
<name>mapreduce.framework.name</name> | |
<value>yarn</value> | |
</property> | |
<property> | |
<name>mapreduce.jobhistory.address</name> | |
<value>gremlin00.beocat.ksu.edu:10020</value> | |
</property> | |
<property> | |
<name>mapreduce.jobhistory.webapp.address</name> | |
<value>gremlin00.beocat.ksu.edu:19888</value> | |
</property> | |
<property> | |
<name>yarn.app.mapreduce.am.staging-dir</name> | |
<value>/staging</value> | |
</property> | |
</configuration> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0"?> | |
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> | |
<configuration> | |
<property> | |
<name>yarn.nodemanager.aux-services</name> | |
<value>mapreduce_shuffle</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.hostname</name> | |
<value>gremlin00.beocat.ksu.edu</value> | |
</property> | |
<property> | |
<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name> | |
<value>org.apache.hadoop.mapred.ShuffleHandler</value> | |
</property> | |
<property> | |
<name>yarn.app.mapreduce.am.staging-dir</name> | |
<value>/staging</value> | |
</property> | |
<property> | |
<name>yarn.log-aggregation-enable</name> | |
<value>true</value> | |
</property> | |
<property> | |
<description>Where to aggregate logs to.</description> | |
<name>yarn.nodemanager.remote-app-log-dir</name> | |
<value>ceph://hobbit01.beocat.ksu.edu:6789/var/log/hadoop-yarn/apps</value> | |
</property> | |
<property> | |
<description>Classpath for typical applications.</description> | |
<name>yarn.application.classpath</name> | |
<value> | |
$HADOOP_CONF_DIR, | |
$HADOOP_COMMON_HOME/*, | |
$HADOOP_COMMON_HOME/lib/*, | |
$HADOOP_HDFS_HOME/*, | |
$HADOOP_HDFS_HOME/lib/*, | |
$HADOOP_MAPRED_HOME/*, | |
$HADOOP_MAPRED_HOME/lib/*, | |
$HADOOP_YARN_HOME/*, | |
$HADOOP_YARN_HOME/lib/* | |
</value> | |
</property> | |
<property> | |
<description>Number of CPU cores that can be allocated for containers.</description> | |
<name>yarn.nodemanager.resource.cpu-vcores</name> | |
<value>12</value> | |
</property> | |
<property> | |
<description>Amount of physical memory, in MB, that can be allocated for containers.</description> | |
<name>yarn.nodemanager.resource.memory-mb</name> | |
<value>65536</value> | |
</property> | |
</configuration> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment