Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
HDP 2.3.4 HA Blueprint
{
"Blueprints": {
"stack_name": "HDP",
"stack_version": "2.3",
"blueprint_name": "HA_blueprint"
},
"host_groups": [
{
"cardinality": "1",
"name": "master_1",
"components": [
{
"name": "ZOOKEEPER_SERVER"
},
{
"name": "OOZIE_SERVER"
},
{
"name": "FALCON_SERVER"
},
{
"name": "HBASE_MASTER"
},
{
"name": "KNOX_GATEWAY"
},
{
"name": "SLIDER"
},
{
"name": "SQOOP"
},
{
"name": "ZOOKEEPER_CLIENT"
},
{
"name": "OOZIE_CLIENT"
},
{
"name": "TEZ_CLIENT"
},
{
"name": "HDFS_CLIENT"
},
{
"name": "YARN_CLIENT"
},
{
"name": "MAPREDUCE2_CLIENT"
},
{
"name": "SPARK_CLIENT"
},
{
"name": "KAFKA_BROKER"
},
{
"name": "FLUME_HANDLER"
},
{
"name": "HIVE_CLIENT"
},
{
"name": "HBASE_CLIENT"
},
{
"name": "FALCON_CLIENT"
},
{
"name": "METRICS_MONITOR"
}
]
},
{
"cardinality": "1",
"name": "master_2",
"components": [
{
"name": "ZOOKEEPER_SERVER"
},
{
"name": "PIG"
},
{
"name": "NIMBUS"
},
{
"name": "STORM_UI_SERVER"
},
{
"name": "NAMENODE"
},
{
"name": "ZKFC"
},
{
"name": "JOURNALNODE"
},
{
"name": "SPARK_JOBHISTORYSERVER"
},
{
"name": "ZOOKEEPER_CLIENT"
},
{
"name": "OOZIE_CLIENT"
},
{
"name": "TEZ_CLIENT"
},
{
"name": "HDFS_CLIENT"
},
{
"name": "YARN_CLIENT"
},
{
"name": "MAPREDUCE2_CLIENT"
},
{
"name": "SPARK_CLIENT"
},
{
"name": "KAFKA_BROKER"
},
{
"name": "FLUME_HANDLER"
},
{
"name": "HIVE_CLIENT"
},
{
"name": "HBASE_CLIENT"
},
{
"name": "FALCON_CLIENT"
},
{
"name": "METRICS_MONITOR"
}
]
},
{
"cardinality": "1",
"name": "master_3",
"components": [
{
"name": "ZOOKEEPER_SERVER"
},
{
"name": "NAMENODE"
},
{
"name": "ZKFC"
},
{
"name": "JOURNALNODE"
},
{
"name": "APP_TIMELINE_SERVER"
},
{
"name": "DRPC_SERVER"
},
{
"name": "OOZIE_SERVER"
},
{
"name": "HISTORYSERVER"
},
{
"name": "ZOOKEEPER_CLIENT"
},
{
"name": "OOZIE_CLIENT"
},
{
"name": "TEZ_CLIENT"
},
{
"name": "HDFS_CLIENT"
},
{
"name": "YARN_CLIENT"
},
{
"name": "MAPREDUCE2_CLIENT"
},
{
"name": "SPARK_CLIENT"
},
{
"name": "KAFKA_BROKER"
},
{
"name": "FLUME_HANDLER"
},
{
"name": "HIVE_CLIENT"
},
{
"name": "HBASE_CLIENT"
},
{
"name": "FALCON_CLIENT"
},
{
"name": "METRICS_MONITOR"
}
]
},
{
"cardinality": "1",
"name": "master_4",
"components": [
{
"name": "ZOOKEEPER_SERVER"
},
{
"name": "RESOURCEMANAGER"
},
{
"name": "JOURNALNODE"
},
{
"name": "HIVE_SERVER"
},
{
"name": "HIVE_METASTORE"
},
{
"name": "WEBHCAT_SERVER"
},
{
"name": "HCAT"
},
{
"name": "MYSQL_SERVER"
},
{
"name": "ATLAS_SERVER"
},
{
"name": "ZOOKEEPER_CLIENT"
},
{
"name": "OOZIE_CLIENT"
},
{
"name": "TEZ_CLIENT"
},
{
"name": "HDFS_CLIENT"
},
{
"name": "YARN_CLIENT"
},
{
"name": "MAPREDUCE2_CLIENT"
},
{
"name": "SPARK_CLIENT"
},
{
"name": "KAFKA_BROKER"
},
{
"name": "FLUME_HANDLER"
},
{
"name": "HIVE_CLIENT"
},
{
"name": "HBASE_CLIENT"
},
{
"name": "FALCON_CLIENT"
},
{
"name": "METRICS_MONITOR"
}
]
},
{
"cardinality": "1",
"name": "master_5",
"components": [
{
"name": "ZOOKEEPER_SERVER"
},
{
"name": "RESOURCEMANAGER"
},
{
"name": "HIVE_SERVER"
},
{
"name": "HIVE_METASTORE"
},
{
"name": "WEBHCAT_SERVER"
},
{
"name": "HCAT"
},
{
"name": "METRICS_COLLECTOR"
},
{
"name": "ZOOKEEPER_CLIENT"
},
{
"name": "OOZIE_CLIENT"
},
{
"name": "TEZ_CLIENT"
},
{
"name": "HDFS_CLIENT"
},
{
"name": "YARN_CLIENT"
},
{
"name": "MAPREDUCE2_CLIENT"
},
{
"name": "SPARK_CLIENT"
},
{
"name": "KAFKA_BROKER"
},
{
"name": "FLUME_HANDLER"
},
{
"name": "HIVE_CLIENT"
},
{
"name": "HBASE_CLIENT"
},
{
"name": "FALCON_CLIENT"
},
{
"name": "METRICS_MONITOR"
}
]
},
{
"name": "workers",
"components": [
{
"name": "DATANODE"
},
{
"name": "NODEMANAGER"
},
{
"name": "SUPERVISOR"
},
{
"name": "HBASE_REGIONSERVER"
},
{
"name": "METRICS_MONITOR"
}
]
}
],
"configurations": [
{
"ams-env": {
"properties": {
"metrics_collector_heapsize": "512m"
}
}
},
{
"hadoop-env": {
"properties": {
"dtnode_heapsize": "2048m"
}
}
},
{
"ams-hbase-env": {
"properties": {
"hbase_master_heapsize": "1024m",
"hbase_regionserver_heapsize": "1024m",
"hbase_master_xmn_size": "128m"
}
}
},
{
"hbase-env": {
"properties": {
"hbase_master_heapsize": "1024m",
"hbase_regionserver_heapsize": "1024m"
}
}
},
{
"yarn-site": {
"properties": {
"yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
"yarn.nodemanager.local-dirs": "/datadisks/disk1/hadoop/yarn/local,/datadisks/disk2/hadoop/yarn/local,/datadisks/disk3/hadoop/yarn/local,/datadisks/disk4/hadoop/yarn/local,/datadisks/disk5/hadoop/yarn/local,/datadisks/disk6/hadoop/yarn/local,/datadisks/disk7/hadoop/yarn/local,/datadisks/disk8/hadoop/yarn/local,/datadisks/disk9/hadoop/yarn/local,/datadisks/disk10/hadoop/yarn/local",
"yarn.resourcemanager.resource-tracker.address": "%HOSTGROUP::master_4%:8025",
"yarn.resourcemanager.hostname": "%HOSTGROUP::master_4%",
"yarn.nodemanager.resource.memory-mb": "32768",
"yarn.scheduler.minimum-allocation-mb": "2048",
"yarn.resourcemanager.address": "%HOSTGROUP::master_4%:8050",
"yarn.resourcemanager.scheduler.address": "%HOSTGROUP::master_4%:8030",
"yarn.resourcemanager.ha.automatic-failover.zk-base-path": "/yarn-leader-election",
"yarn.scheduler.maximum-allocation-mb": "32768",
"yarn.resourcemanager.ha.enabled": "true",
"hadoop.registry.rm.enabled": "false",
"yarn.timeline-service.webapp.address": "%HOSTGROUP::master_4%:8188",
"yarn.resourcemanager.webapp.https.address": "%HOSTGROUP::master_4%:8090",
"yarn.timeline-service.address": "%HOSTGROUP::master_4%:10200",
"yarn.resourcemanager.recovery.enabled": "true",
"yarn.resourcemanager.hostname.rm1": "%HOSTGROUP::master_4%",
"yarn.resourcemanager.hostname.rm2": "%HOSTGROUP::master_5%",
"hadoop.registry.zk.quorum": "%HOSTGROUP::master_1%:2181,%HOSTGROUP::master_2%:2181,%HOSTGROUP::master_3%:2181,%HOSTGROUP::master_4%:2181,%HOSTGROUP::master_5%:2181",
"yarn.resourcemanager.cluster-id": "mycluster",
"yarn.resourcemanager.webapp.address": "%HOSTGROUP::master_4%:8088",
"yarn.timeline-service.leveldb-timeline-store.path": "/datadisks/disk1/hadoop/yarn/timeline",
"yarn.nodemanager.log-dirs": "/datadisks/disk1/hadoop/yarn/log,/datadisks/disk2/hadoop/yarn/log,/datadisks/disk3/hadoop/yarn/log,/datadisks/disk4/hadoop/yarn/log,/datadisks/disk5/hadoop/yarn/log,/datadisks/disk6/hadoop/yarn/log,/datadisks/disk7/hadoop/yarn/log,/datadisks/disk8/hadoop/yarn/log,/datadisks/disk9/hadoop/yarn/log,/datadisks/disk10/hadoop/yarn/log",
"yarn.log.server.url": "http://%HOSTGROUP::master_3%:19888/jobhistory/logs",
"yarn.resourcemanager.ha.rm-ids": "rm1,rm2",
"yarn.resourcemanager.admin.address": "%HOSTGROUP::master_4%:8141",
"yarn.timeline-service.webapp.https.address": "%HOSTGROUP::master_4%:8190"
}
}
},
{
"core-site": {
"properties": {
"hadoop.proxyuser.hcat.groups": "users",
"hadoop.proxyuser.oozie.groups": "*",
"hadoop.proxyuser.hcat.hosts": "%HOSTGROUP::master_4%,%HOSTGROUP::master_5%",
"hadoop.proxyuser.hive.groups": "users",
"io.compression.codec.lzo.class": "com.hadoop.compression.lzo.LzoCodec",
"hadoop.proxyuser.hive.hosts": "%HOSTGROUP::master_4%,%HOSTGROUP::master_5%",
"ha.zookeeper.quorum": "%HOSTGROUP::master_1%:2181,%HOSTGROUP::master_2%:2181,%HOSTGROUP::master_3%:2181,%HOSTGROUP::master_4%:2181,%HOSTGROUP::master_5%:2181",
"io.compression.codecs": "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec",
"hadoop.proxyuser.oozie.hosts": "%HOSTGROUP::master_1%, %HOSTGROUP::master_3%",
"fs.defaultFS": "hdfs://mycluster"
}
}
},
{
"tez-site": {
"properties": {
"tez.am.java.opts": "-server -Xmx1638m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC",
"tez.am.resource.memory.mb": "2048"
}
}
},
{
"mapred-site": {
"properties": {
"mapreduce.reduce.java.opts": "-Xmx1638m",
"mapreduce.map.memory.mb": "2048",
"mapreduce.map.output.compress": "true",
"mapreduce.task.io.sort.mb": "819",
"yarn.app.mapreduce.am.command-opts": "-Xmx1638m -Dhdp.version=${hdp.version}",
"mapreduce.reduce.memory.mb": "2048",
"mapreduce.map.java.opts": "-Xmx1638m",
"yarn.app.mapreduce.am.resource.mb": "2048"
}
}
},
{
"hdfs-site": {
"properties": {
"dfs.namenode.https-address.mycluster.nn2": "%HOSTGROUP::master_3%:50470",
"dfs.namenode.rpc-address.mycluster.nn2": "%HOSTGROUP::master_3%:8020",
"dfs.namenode.https-address.mycluster.nn1": "%HOSTGROUP::master_2%:50470",
"dfs.datanode.data.dir": "/datadisks/disk1/hadoop/hdfs/data,/datadisks/disk2/hadoop/hdfs/data,/datadisks/disk3/hadoop/hdfs/data,/datadisks/disk4/hadoop/hdfs/data,/datadisks/disk5/hadoop/hdfs/data,/datadisks/disk6/hadoop/hdfs/data,/datadisks/disk7/hadoop/hdfs/data,/datadisks/disk8/hadoop/hdfs/data,/datadisks/disk9/hadoop/hdfs/data,/datadisks/disk10/hadoop/hdfs/data",
"dfs.namenode.http-address": "%HOSTGROUP::master_2%:50070",
"dfs.ha.namenodes.mycluster": "nn1,nn2",
"dfs.namenode.rpc-address.mycluster.nn1": "%HOSTGROUP::master_2%:8020",
"dfs.namenode.https-address": "%HOSTGROUP::master_2%:50470",
"dfs.datanode.failed.volumes.tolerated": "6",
"dfs.client.failover.proxy.provider.mycluster": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
"dfs.nameservices": "mycluster",
"dfs.namenode.http-address.mycluster.nn2": "%HOSTGROUP::master_3%:50070",
"dfs.namenode.http-address.mycluster.nn1": "%HOSTGROUP::master_2%:50070",
"dfs.namenode.name.dir": "/datadisks/disk1/hadoop/hdfs/namenode,/datadisks/disk2/hadoop/hdfs/namenode,/datadisks/disk3/hadoop/hdfs/namenode,/datadisks/disk4/hadoop/hdfs/namenode,/datadisks/disk5/hadoop/hdfs/namenode,/datadisks/disk6/hadoop/hdfs/namenode,/datadisks/disk7/hadoop/hdfs/namenode,/datadisks/disk8/hadoop/hdfs/namenode",
"dfs.ha.fencing.methods": "shell(/bin/true)",
"dfs.namenode.checkpoint.dir": "/datadisks/disk1/hadoop/hdfs/namesecondary",
"dfs.namenode.shared.edits.dir": "qjournal://%HOSTGROUP::master_2%:8485;%HOSTGROUP::master_3%:8485;%HOSTGROUP::master_4%:8485/mycluster",
"dfs.ha.automatic-failover.enabled": "true"
}
}
},
{
"hbase-site": {
"properties": {
"hbase.tmp.dir": "/datadisks/disk1/hadoop/hbase"
}
}
},
{
"storm-site": {
"properties": {
"storm.local.dir": "/datadisks/disk1/hadoop/storm"
}
}
},
{
"falcon-startup.properties": {
"properties": {
"*.config.store.uri": "file:///datadisks/disk1/hadoop/falcon/store"
}
}
},
{
"hive-site": {
"properties": {
"hive.tez.container.size": "1024",
"hive.tez.java.opts": "-server -Xmx820m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps",
"hive.metastore.uris": "thrift://%HOSTGROUP::master_4%:9083,thrift://%HOSTGROUP::master_5%:9083",
"hive.auto.convert.join.noconditionaltask.size": "357564416"
}
}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.