Capacity scheduler configuration

capacity-scheduler.xml

  <configuration>
    
    <property>
      <!-- Maximum resources to allocate to application masters
           If this is too high application masters can crowd out actual work -->
      <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
      <value>0.25</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.maximum-applications</name>
      <value>10000</value>
    </property>
    
    <property>
      <!-- Yarn knows where the data is local, this is the number of *times*
           to try to schedule on a local node before going for rack-local -->
      <name>yarn.scheduler.capacity.node-locality-delay</name>
      <value>40</value>
    </property>
    
    <property>
      <!-- This allows users to pick which queue to submit to.
            Unless you are genuinely multi-tenant this will make your life easier. -->
      <name>yarn.scheduler.capacity.queue-mappings-override.enable</name>
      <value>true</value>
    </property>
    
    <!-- By default CPUs are ignored for scheduling purposes. 
    See https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_yarn_resource_mgt/content/ref-7a0a1ca2-72d6-4f3e-82bd-b600e9905acf.1.html -->
    <property>
      <name>yarn.scheduler.capacity.resource-calculator</name>
      <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
    </property>
    
    <!-- Let the root queue access anything -->
    <property>
      <name>yarn.scheduler.capacity.root.accessible-node-labels</name>
      <value>*</value>
    </property>
    
    <!-- Root queue can access 100% of data label -->
    <property>
      <name>yarn.scheduler.capacity.root.accessible-node-labels.data.capacity</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.accessible-node-labels.data.maximum-capacity</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.acl_administer_queue</name>
      <value>*</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.capacity</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.accessible-node-labels</name>
      <value>*</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
      <value>*</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.capacity</name>
      <value>98</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.maximum-am-resource-percent</name>
      <value>0.25</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.minimum-user-limit-percent</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.ordering-policy</name>
      <value>fair</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.ordering-policy.fair.enable-size-based-weight</name>
      <value>false</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.state</name>
      <value>RUNNING</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
      <value>1</value>
    </property>
    
    <!-- Pin interactive queue to data label -->
    <property>
      <name>yarn.scheduler.capacity.root.interactive.accessible-node-labels</name>
      <value>data</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.accessible-node-labels.data.capacity</name>
      <value>20</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.accessible-node-labels.data.maximum-capacity</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.acl_administer_queue</name>
      <value>*</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.acl_submit_applications</name>
      <value>*</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.capacity</name>
      <value>1</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.default-node-label-expression</name>
      <value>data</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.maximum-am-resource-percent</name>
      <value>0.25</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.maximum-applications</name>
      <value>2000</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.maximum-capacity</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.minimum-user-limit-percent</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.state</name>
      <value>RUNNING</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.interactive.user-limit-factor</name>
      <value>35</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.maximum-capacity</name>
      <value>100</value>
    </property>
    
    <!-- Pin noninteractive queue to data label -->
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.accessible-node-labels</name>
      <value>data</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.accessible-node-labels.data.capacity</name>
      <value>80</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.accessible-node-labels.data.maximum-capacity</name>
      <value>80</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.acl_submit_applications</name>
      <value>*</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.capacity</name>
      <value>1</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.default-node-label-expression</name>
      <value>data</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.maximum-am-resource-percent</name>
      <value>0.5</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.maximum-applications</name>
      <value>8000</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.maximum-capacity</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.minimum-user-limit-percent</name>
      <value>100</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.state</name>
      <value>RUNNING</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.noninteractive.user-limit-factor</name>
      <value>77</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.queues</name>
      <value>default,interactive,noninteractive</value>
    </property>
    
    <property>
      <name>yarn.scheduler.capacity.root.user-limit-factor</name>
      <value>0.5</value>
    </property>
    
  </configuration>