Skip to content

Instantly share code, notes, and snippets.

@avivl
Last active Feb 4, 2018
Embed
What would you like to do?
def calc_how_many(self):
"""
Calculate how many new nodes of each type we need.
:return:
"""
# No allocated memory so we don't need any workers above the
# bare minimum
if self.scale_to != -1:
self.total = self.min_instances
logging.debug('No allocated memory lets go down! New workers %s'
' New preemptibel', self.total)
return
# no more memory lets get some nodes. claculate how many memory each
# node uses. Then calculate how many nodes we need by memory
# consumption
if self.dataproc.get_yarn_memory_available_percentage() == 0:
yarn_memory_mb_allocated, yarn_memory_mb_pending = \
self.dataproc.get_memory_data()
ratio = float(
int(yarn_memory_mb_allocated) / int(self.current_nodes))
factor = float(int(yarn_memory_mb_pending) / ratio)
self.total = int(self.current_nodes * factor)
logging.debug(
'yarn_memory_mb_allocated %s pending %s ratio %s factor %s'
' current %s total %s', yarn_memory_mb_allocated,
yarn_memory_mb_pending, ratio, factor, self.current_nodes,
self.total)
logging.debug('No More Mem! New workers %s prev %s', self.total,
self.current_nodes)
return
# pending containers are waiting....
if self.containerpendingratio != -1:
yarn_containers_allocated, yarn_containers_pending = \
self.dataproc.get_container_data()
ratio = float(
int(yarn_containers_allocated) / int(self.current_nodes))
factor = float(int(yarn_containers_pending) / ratio)
self.total = int(self.current_nodes * factor)
logging.debug(
'yarn_containers_allocated %s pending %s ratio %s factor %s'
' current %s total %s', yarn_containers_allocated,
yarn_containers_pending, ratio, factor, self.current_nodes,
self.total)
logging.debug('Need more containers! New workers %s prev %s',
self.total, self.current_nodes)
return
self.calc_scale()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment