gistfile1.txt

Here is the recipe that worked:

#
# Cookbook Name:: provisioning_pds_time_series
# Recipe:: cluster
#
# Copyright (c) 2015 The Authors, All Rights Reserved.

cluster_name = node['provisioning']['cluster_name']
data = node[cookbook_name][cluster_name]['cluster']

Chef::Log.info "Provisioning #{ cluster_name } cluster"

machine_batch do
  1.upto(data['node_count']) do |i|
    machine "dataservices-#{cluster_name}-#{i}" do
      chef_environment node['provisioning']['environment']
      machine_options(
        use_priviate_ip_for_ssh: true,
        ssh_username: data['ssh_username'],
        bootstrap_options: {
          key_name: data['key_pair'],
          image_id: data['ami'],
          security_group_ids: data['security_group_ids'],
          subnet_id: data['subnet_ids'].first,
          instance_type: data['instance_type']
        }
      )
      attributes(
        pds_kairosdb: {
          cluster_name: cluster_name,
          java: {
            xmx: data['pds_kairosdb']['java']['xmx'],
            xms: data['pds_kairosdb']['java']['xms'],
            maxnewsize: data['pds_kairosdb']['java']['maxnewsize']
          },
          version: data['pds_kairosdb']['version'],
          newrelic: {
            install: data['pds_kairosdb']['newrelic']['install'],
            version: data['pds_kairosdb']['newrelic']['version'],
            license: data['pds_kairosdb']['newrelic']['license'],
            app_name: data['pds_kairosdb']['newrelic']['app_name']
          }
        },
        pds_cassandra: {
          search_cluster_name: data['pds_cassandra']['search_cluster_name']
        }
      )
      recipe 'pds_kairosdb'
    end
    data['subnet_ids'].rotate!
  end
end

machine "dataservices-#{cluster_name}-lb" do
  chef_environment node['provisioning']['environment']
  machine_options(
    use_priviate_ip_for_ssh: true,
    ssh_username: data['ssh_username'],
    bootstrap_options: {
      key_name: data['key_pair'],
      image_id: data['ami'],
      security_group_ids: data['security_group_ids'],
      subnet_id: data['subnet_ids'].first,
      instance_type: data['instance_type']
    }
  )
  attributes(
    pds_time_series: {
      search_cluster_name: cluster_name
    }
  )
  recipe 'pds_time_series_lb'
end

Here is the recipe that has broken it:

#
# Cookbook Name:: provisioning_pds_time_series
# Recipe:: cluster
#
# Copyright (c) 2015 The Authors, All Rights Reserved.

cluster_name = node['provisioning']['cluster_name']
cluster_data = node[cookbook_name][cluster_name]['cluster']
cluster_attributes = node[cookbook_name][cluster_name]['attributes'].to_hash

cluster_attributes['pds_kairosdb']['cluster_name'] = cluster_name

Chef::Log.info "Provisioning #{ cluster_name } cluster"

machine_batch do
  1.upto(cluster_data['node_count']) do |i|
    machine "dataservices-#{cluster_name}-#{i}" do
      chef_environment node['provisioning']['environment']
      machine_options(
        use_priviate_ip_for_ssh: true,
        ssh_username: cluster_data['ssh_username'],
        bootstrap_options: {
          key_name: cluster_data['key_pair'],
          image_id: cluster_data['ami'],
          security_group_ids: cluster_data['security_group_ids'],
          subnet_id: cluster_data['subnet_ids'].first,
          instance_type: cluster_data['instance_type']
        }
      )
      attributes(cluster_attributes)
      recipe 'pds_kairosdb'
    end
    cluster_data['subnet_ids'].rotate!
  end
end

machine "dataservices-#{cluster_name}-lb" do
  chef_environment node['provisioning']['environment']
  machine_options(
    use_priviate_ip_for_ssh: true,
    ssh_username: cluster_data['ssh_username'],
    bootstrap_options: {
      key_name: cluster_data['key_pair'],
      image_id: cluster_data['ami'],
      security_group_ids: cluster_data['security_group_ids'],
      subnet_id: cluster_data['subnet_ids'].first,
      instance_type: cluster_data['instance_type']
    }
  )
  attributes(
    pds_time_series: {
      search_cluster_name: cluster_name
    }
  )
  recipe 'pds_time_series_lb'
end

Here is the output from the chef run:

Starting Chef Client, version 12.3.0
[2015-09-02T18:30:19+00:00] WARN: Run List override has been provided.
[2015-09-02T18:30:19+00:00] WARN: Original Run List: [recipe[ice_base::default], role[provisioning]]
[2015-09-02T18:30:19+00:00] WARN: Overridden Run List: [role[ds-ts-time-series-development], recipe[provisioning_pds_time_series]]
resolving cookbooks for run list: ["provisioning_pds_time_series"]
Synchronizing Cookbooks:
  - provisioning_pds_time_series
Compiling Cookbooks...
Recipe: provisioning_pds_time_series::provision
  * chef_gem[chef-provisioning-aws] action installWARN: Unresolved specs during Gem::Specification.reset:
      nokogiri (>= 1.4.0, ~> 1.5)
WARN: Clearing out unresolved specs.
Please report a bug if this causes problems.
 (up to date)
  Converging 3 resources
  * chef_gem[chef-provisioning-aws] action installWARN: Unresolved specs during Gem::Specification.reset:
      mini_portile (~> 0.6.0)
WARN: Clearing out unresolved specs.
Please report a bug if this causes problems.
 (up to date)
Recipe: provisioning_pds_time_series::cluster
  * machine_batch[default] action converge
    - creating machines dataservices-ds-ts-time-series-development-1, dataservices-ds-ts-time-series-development-3 on aws:IAM:us-east-1
    -   key_name: "ds-ts"
    -   image_id: "ami-d05e75b8"
    -   security_group_ids: ["sg-a59454c2"]
    -   subnet_id: "subnet-602b8f4b"
    -   instance_type: "c3.xlarge"
    - creating machine dataservices-ds-ts-time-series-development-2 on aws:IAM:us-east-1
    -   key_name: "ds-ts"
    -   image_id: "ami-d05e75b8"
    -   security_group_ids: ["sg-a59454c2"]
    -   subnet_id: "subnet-f465fb83"
    -   instance_type: "c3.xlarge"
    - machine dataservices-ds-ts-time-series-development-3 created as i-2bcfc980 on aws:IAM:us-east-1
    - [dataservices-ds-ts-time-series-development-3] create node dataservices-ds-ts-time-series-development-3 at https://example.com/organizations/cf-data-services
    - [dataservices-ds-ts-time-series-development-3]   update chef_environment from "_default" to "development"
    - [dataservices-ds-ts-time-series-development-3]   add normal.pds_cassandra = {"search_cluster_name"=>"ds-ts-cassandra-development"}
    - [dataservices-ds-ts-time-series-development-3]   add normal.pds_kairosdb = {"java"=>{"xmx"=>"3G", "xms"=>"3G", "maxnewsize"=>"1G"}, "version"=>"1.0.2-SNAPSHOT", "newrelic"=>{"install"=>true, "version"=>"3.18.0", "license"=>"44e99c4dec77edb10c604eb7db719610ae4c26eb", "app_name"=>"time-series-store-query-development"}, "cluster_name"=>"ds-ts-time-series-development"}
    - [dataservices-ds-ts-time-series-development-3]   add normal.tags = nil
    - [dataservices-ds-ts-time-series-development-3]   add normal.chef_provisioning = {"reference"=>{"driver_version"=>"1.2.1", "allocated_at"=>"2015-09-02 18:30:29 UTC", "host_node"=>"https://example.com/organizations/cf-data-services/nodes/", "image_id"=>"ami-d05e75b8", "instance_id"=>"i-2bcfc980", "key_name"=>"ds-ts", "ssh_username"=>"ubuntu"}, "driver_url"=>"aws:IAM:us-east-1"}
    - [dataservices-ds-ts-time-series-development-3]   update run_list from [] to ["recipe[pds_kairosdb]"]
    - machine dataservices-ds-ts-time-series-development-1 created as i-2acfc981 on aws:IAM:us-east-1
    - [dataservices-ds-ts-time-series-development-1] create node dataservices-ds-ts-time-series-development-1 at https://example.com/organizations/cf-data-services
    - [dataservices-ds-ts-time-series-development-1]   update chef_environment from "_default" to "development"
    - [dataservices-ds-ts-time-series-development-1]   add normal.pds_cassandra = {"search_cluster_name"=>"ds-ts-cassandra-development"}
    - [dataservices-ds-ts-time-series-development-1]   add normal.pds_kairosdb = {"java"=>{"xmx"=>"3G", "xms"=>"3G", "maxnewsize"=>"1G"}, "version"=>"1.0.2-SNAPSHOT", "newrelic"=>{"install"=>true, "version"=>"3.18.0", "license"=>"44e99c4dec77edb10c604eb7db719610ae4c26eb", "app_name"=>"time-series-store-query-development"}, "cluster_name"=>"ds-ts-time-series-development"}
    - [dataservices-ds-ts-time-series-development-1]   add normal.tags = nil
    - [dataservices-ds-ts-time-series-development-1]   add normal.chef_provisioning = {"reference"=>{"driver_version"=>"1.2.1", "allocated_at"=>"2015-09-02 18:30:30 UTC", "host_node"=>"https://example.com/organizations/cf-data-services/nodes/", "image_id"=>"ami-d05e75b8", "instance_id"=>"i-2acfc981", "key_name"=>"ds-ts", "ssh_username"=>"ubuntu"}, "driver_url"=>"aws:IAM:us-east-1"}
    - [dataservices-ds-ts-time-series-development-1]   update run_list from [] to ["recipe[pds_kairosdb]"]
    - machine dataservices-ds-ts-time-series-development-2 created as i-5aac4b8f on aws:IAM:us-east-1
    - [dataservices-ds-ts-time-series-development-2] create node dataservices-ds-ts-time-series-development-2 at https://example.com/organizations/cf-data-services
    - [dataservices-ds-ts-time-series-development-2]   update chef_environment from "_default" to "development"
    - [dataservices-ds-ts-time-series-development-2]   add normal.pds_cassandra = {"search_cluster_name"=>"ds-ts-cassandra-development"}
    - [dataservices-ds-ts-time-series-development-2]   add normal.pds_kairosdb = {"java"=>{"xmx"=>"3G", "xms"=>"3G", "maxnewsize"=>"1G"}, "version"=>"1.0.2-SNAPSHOT", "newrelic"=>{"install"=>true, "version"=>"3.18.0", "license"=>"44e99c4dec77edb10c604eb7db719610ae4c26eb", "app_name"=>"time-series-store-query-development"}, "cluster_name"=>"ds-ts-time-series-development"}
    - [dataservices-ds-ts-time-series-development-2]   add normal.tags = nil
    - [dataservices-ds-ts-time-series-development-2]   add normal.chef_provisioning = {"reference"=>{"driver_version"=>"1.2.1", "allocated_at"=>"2015-09-02 18:30:30 UTC", "host_node"=>"https://example.com/organizations/cf-data-services/nodes/", "image_id"=>"ami-d05e75b8", "instance_id"=>"i-5aac4b8f", "key_name"=>"ds-ts", "ssh_username"=>"ubuntu"}, "driver_url"=>"aws:IAM:us-east-1"}
    - [dataservices-ds-ts-time-series-development-2]   update run_list from [] to ["recipe[pds_kairosdb]"]
    - [dataservices-ds-ts-time-series-development-1] waiting for dataservices-ds-ts-time-series-development-1 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-1] been waiting 0/120 -- sleeping 10 seconds for dataservices-ds-ts-time-series-development-1 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-3] waiting for dataservices-ds-ts-time-series-development-3 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-3] been waiting 0/120 -- sleeping 10 seconds for dataservices-ds-ts-time-series-development-3 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-2] waiting for dataservices-ds-ts-time-series-development-2 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-2] been waiting 0/120 -- sleeping 10 seconds for dataservices-ds-ts-time-series-development-2 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-1] been waiting 10/120 -- sleeping 10 seconds for dataservices-ds-ts-time-series-development-1 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-3] been waiting 10/120 -- sleeping 10 seconds for dataservices-ds-ts-time-series-development-3 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-2] been waiting 10/120 -- sleeping 10 seconds for dataservices-ds-ts-time-series-development-2 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-1] been waiting 20/120 -- sleeping 10 seconds for dataservices-ds-ts-time-series-development-1 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-3] been waiting 20/120 -- sleeping 10 seconds for dataservices-ds-ts-time-series-development-3 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-2] been waiting 20/120 -- sleeping 10 seconds for dataservices-ds-ts-time-series-development-2 (i-5aac4b8f on aws:IAM:us-east-1) to be ready ...
    - [dataservices-ds-ts-time-series-development-1] dataservices-ds-ts-time-series-development-1 is now ready
    - [dataservices-ds-ts-time-series-development-3] dataservices-ds-ts-time-series-development-3 is now ready
    - [dataservices-ds-ts-time-series-development-2] dataservices-ds-ts-time-series-development-2 is now ready