calc_how_many.py

   def calc_how_many(self):
        """
        Calculate how  many new nodes of each type we need.

        :return:
        """
        # No allocated memory so we don't need any workers above the
        # bare minimum
        if self.scale_to != -1:
            self.total = self.min_instances
            logging.debug('No allocated memory lets go down! New workers %s'
                          ' New preemptibel', self.total)
            return
        # no more memory lets get some  nodes. claculate how many memory each
        # node uses. Then calculate how many nodes we need by memory
        # consumption
        if self.dataproc.get_yarn_memory_available_percentage() == 0:
            yarn_memory_mb_allocated, yarn_memory_mb_pending = \
                self.dataproc.get_memory_data()
            ratio = float(
                int(yarn_memory_mb_allocated) / int(self.current_nodes))
            factor = float(int(yarn_memory_mb_pending) / ratio)
            self.total = int(self.current_nodes * factor)
            logging.debug(
                'yarn_memory_mb_allocated %s pending %s ratio %s factor %s'
                ' current %s total %s', yarn_memory_mb_allocated,
                yarn_memory_mb_pending, ratio, factor, self.current_nodes,
                self.total)
            logging.debug('No More Mem! New workers %s  prev %s', self.total,
                          self.current_nodes)
            return

        # pending containers are waiting....
        if self.containerpendingratio != -1:
            yarn_containers_allocated, yarn_containers_pending = \
                self.dataproc.get_container_data()
            ratio = float(
                int(yarn_containers_allocated) / int(self.current_nodes))
            factor = float(int(yarn_containers_pending) / ratio)
            self.total = int(self.current_nodes * factor)
            logging.debug(
                'yarn_containers_allocated %s pending %s ratio %s factor %s'
                ' current %s total %s', yarn_containers_allocated,
                yarn_containers_pending, ratio, factor, self.current_nodes,
                self.total)
            logging.debug('Need more containers! New workers %s  prev %s',
                          self.total, self.current_nodes)
            return

        self.calc_scale()