updated to whirr-0.3.0

gistfile1.diff

diff -Naur whirr-0.3.0-incubating/contrib/python/src/py/hadoop/cloud/cli.py whirr-0.3.0-incubating-backtype/contrib/python/src/py/hadoop/cloud/cli.py
--- whirr-0.3.0-incubating/contrib/python/src/py/hadoop/cloud/cli.py	2011-01-15 23:03:44.000000000 -0800
+++ whirr-0.3.0-incubating-backtype/contrib/python/src/py/hadoop/cloud/cli.py	2011-02-15 11:51:49.000000000 -0800
@@ -296,7 +296,7 @@
                          opt.get('user_data_file'),
                          opt.get('availability_zone'), opt.get('user_packages'),
                          opt.get('auto_shutdown'), opt.get('env'),
-                         opt.get('security_group'))
+                         opt.get('security_group'), opt.get('spot_price'))
     service.launch_master(template, config_dir, opt.get('client_cidr'))
 
   elif command == 'launch-slaves':
@@ -311,7 +311,7 @@
                          opt.get('user_data_file'),
                          opt.get('availability_zone'), opt.get('user_packages'),
                          opt.get('auto_shutdown'), opt.get('env'),
-                         opt.get('security_group'))
+                         opt.get('security_group'), opt.get('spot_price'))
     service.launch_slaves(template)
 
   elif command == 'launch-cluster':
@@ -333,7 +333,7 @@
                          opt.get('user_data_file'),
                          opt.get('availability_zone'), opt.get('user_packages'),
                          opt.get('auto_shutdown'), opt.get('env'),
-                         opt.get('security_group')),
+                         opt.get('security_group'), opt.get('spot_price')),
         InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
                          get_image_id(service.cluster, opt),
                          opt.get('instance_type'), opt.get('key_name'),
@@ -341,7 +341,7 @@
                          opt.get('user_data_file'),
                          opt.get('availability_zone'), opt.get('user_packages'),
                          opt.get('auto_shutdown'), opt.get('env'),
-                         opt.get('security_group')),
+                         opt.get('security_group'), opt.get('spot_price')),
                          ]
     elif len(args) > 2 and len(args) % 2 == 0:
       print_usage(sys.argv[0])
@@ -358,7 +358,7 @@
                            opt.get('availability_zone'),
                            opt.get('user_packages'),
                            opt.get('auto_shutdown'), opt.get('env'),
-                           opt.get('security_group')))
+                           opt.get('security_group'), opt.get('spot_price')))
 
     service.launch_cluster(instance_templates, config_dir,
                            opt.get('client_cidr'))
diff -Naur whirr-0.3.0-incubating/contrib/python/src/py/hadoop/cloud/providers/ec2.py whirr-0.3.0-incubating-backtype/contrib/python/src/py/hadoop/cloud/providers/ec2.py
--- whirr-0.3.0-incubating/contrib/python/src/py/hadoop/cloud/providers/ec2.py	2011-01-15 23:03:44.000000000 -0800
+++ whirr-0.3.0-incubating-backtype/contrib/python/src/py/hadoop/cloud/providers/ec2.py	2011-02-15 22:36:41.000000000 -0800
@@ -243,6 +243,46 @@
       sys.stdout.write(".")
       sys.stdout.flush()
       time.sleep(1)
+    
+  def launch_spot_instances(self, roles, price, number, image_id, size_id,
+                       instance_user_data, **kwargs):
+    
+    for role in roles:
+      self._check_role_name(role)  
+      self._create_groups(role)
+      
+    user_data = instance_user_data.read_as_gzip_stream()
+    security_groups = self._get_group_names(roles) + kwargs.get('security_groups', [])
+    
+    spot_request = self.ec2Connection.request_spot_instances(price=price, image_id=image_id,
+      count=number, type='one-time', valid_from=None, valid_until=None,
+      launch_group=kwargs.get('launch_group', None),
+      availability_zone_group=kwargs.get('availability_zone_group', None),
+      key_name=kwargs.get('key_name', None),
+      security_groups=security_groups, user_data=user_data,
+      instance_type=size_id, placement=kwargs.get('placement', None))
+    spot_instance_request_ids = [request.id for request in spot_request]
+    instance_ids = self.wait_for_spot_instances(spot_instance_request_ids)
+    return instance_ids
+
+  def wait_for_spot_instances(self, request_ids, timeout=1200):
+    start_time = time.time()
+    while True:
+      if (time.time() - start_time >= timeout):
+        raise TimeoutException()
+      if self._all_spot_requests_started(self.ec2Connection.get_all_spot_instance_requests(request_ids)):
+        instance_ids = [request.instance_id for request in self.ec2Connection.get_all_spot_instance_requests(request_ids)]
+        if self._all_started(self.ec2Connection.get_all_instances(instance_ids)):
+          return instance_ids
+      sys.stdout.write(".")
+      sys.stdout.flush()
+      time.sleep(1)
+
+  def _all_spot_requests_started(self, requests):
+    for request in requests:
+      if request.state != "active":
+        return False
+    return True
 
   def _all_started(self, reservations):
     for res in reservations:
@@ -255,6 +295,9 @@
     instances = self._get_instances(self._get_cluster_group_name(), "running")
     if instances:
       self.ec2Connection.terminate_instances([i.id for i in instances])
+      spot_instance_request_ids = map(lambda x: x.spot_instance_request_id, filter(lambda x: x.spot_instance_request_id is not None, instances))
+      if spot_instance_request_ids:
+          self.ec2Connection.cancel_spot_instance_requests(spot_instance_request_ids)
 
   def delete(self):
     """
diff -Naur whirr-0.3.0-incubating/contrib/python/src/py/hadoop/cloud/service.py whirr-0.3.0-incubating-backtype/contrib/python/src/py/hadoop/cloud/service.py
--- whirr-0.3.0-incubating/contrib/python/src/py/hadoop/cloud/service.py	2011-01-15 23:03:44.000000000 -0800
+++ whirr-0.3.0-incubating-backtype/contrib/python/src/py/hadoop/cloud/service.py	2011-02-15 16:41:31.000000000 -0800
@@ -52,7 +52,7 @@
                      key_name, public_key, private_key,
                      user_data_file_template=None, placement=None,
                      user_packages=None, auto_shutdown=None, env_strings=[],
-                     security_groups=[]):
+                     security_groups=[], spot_price=None):
     self.roles = roles
     self.number = number
     self.image_id = image_id
@@ -66,6 +66,7 @@
     self.auto_shutdown = auto_shutdown
     self.env_strings = env_strings
     self.security_groups = security_groups
+    self.spot_price = spot_price
 
   def add_env_strings(self, env_strings):
     new_env_strings = list(self.env_strings or [])
@@ -210,12 +211,21 @@
       "EBS_MAPPINGS": ebs_mappings,
     }) }
     instance_user_data = InstanceUserData(user_data_file_template, replacements)
-    instance_ids = self.cluster.launch_instances(it.roles, it.number, it.image_id,
-                                            it.size_id,
-                                            instance_user_data,
-                                            key_name=it.key_name,
-                                            public_key=it.public_key,
-                                            placement=it.placement)
+
+    if it.spot_price is None:
+        instance_ids = self.cluster.launch_instances(it.roles, it.number, it.image_id,
+                                                it.size_id,
+                                                instance_user_data,
+                                                key_name=it.key_name,
+                                                public_key=it.public_key,
+                                                placement=it.placement)
+    else:
+        instance_ids = self.cluster.launch_spot_instances(it.roles, it.spot_price, it.number, it.image_id,
+                                                it.size_id,
+                                                instance_user_data,
+                                                key_name=it.key_name,
+                                                public_key=it.public_key,
+                                                placement=it.placement)
     print "Waiting for %s instances in role %s to start" % \
       (it.number, ",".join(it.roles))
     try:
diff -Naur whirr-0.3.0-incubating/contrib/python/src/py/hadoop-ec2 whirr-0.3.0-incubating-backtype/contrib/python/src/py/hadoop-ec2
--- whirr-0.3.0-incubating/contrib/python/src/py/hadoop-ec2	2011-01-15 23:03:44.000000000 -0800
+++ whirr-0.3.0-incubating-backtype/contrib/python/src/py/hadoop-ec2	2011-02-15 12:00:28.000000000 -0800
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.5
+#!/usr/bin/env python
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with