gistfile1.txt

Date: Sat, 28 Aug 2010 08:00:03 +0000 (UTC)
From: Cron Daemon <root@ip-10-204-107-97.ec2.internal>
To: root@ip-10-204-107-97.ec2.internal
Subject: Cron <root@ip-10-204-107-97> chef-solo -j /etc/chef/chef-solo-database-snapshot.json -c /etc/chef/chef-solo-database-snapshot.rb

[Sat, 28 Aug 2010 08:00:03 +0000] INFO: Setting the run_list to ["recipe[database::snapshot]"] from JSON
[Sat, 28 Aug 2010 08:00:03 +0000] INFO: Starting Chef Run (Version 0.9.8)
[Sat, 28 Aug 2010 08:00:03 +0000] INFO: Overriding duplicate definition pear_module, new found in /var/chef/cache/cookbooks/php/definitions/pear.rb
[Sat, 28 Aug 2010 08:00:03 +0000] INFO: Overriding duplicate definition pear_channel, new found in /var/chef/cache/cookbooks/php/definitions/pear_channel.rb
[Sat, 28 Aug 2010 08:00:03 +0000] INFO: mysql_database: flushing tables with read lock
warning: peer certificate won't be verified in this SSL session
warning: peer certificate won't be verified in this SSL session
[Sat, 28 Aug 2010 08:00:04 +0000] INFO: Ran execute[xfs freeze] successfully
[Sat, 28 Aug 2010 08:00:04 +0000] INFO: New RightAws::Ec2 using shared connections mode
[Sat, 28 Aug 2010 08:00:04 +0000] INFO: Opening new HTTPS connection to ec2.amazonaws.com:443
[Sat, 28 Aug 2010 08:00:08 +0000] INFO: Checking for old snapshots
[Sat, 28 Aug 2010 08:00:14 +0000] WARN: ##### RightAws::Ec2 returned an error: 500 Internal Server Error
<?xml version="1.0"?>
<Response><Errors><Error><Code>InternalError</Code><Message>An internal error has occurred</Message></Error></Errors><RequestID>92d64509-5660-4fed-8271-ca2d50dd6b48</RequestID></Response> #####
[Sat, 28 Aug 2010 08:00:14 +0000] WARN: ##### RightAws::Ec2 request:
+https://ec2.amazonaws.com:443/?AWSAccessKeyId=AKIAIVTL6LH4Q42UURGA&Action=DescribeSnapshots&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2010-08-28T08%3A00%3A08.000Z&Version=2009-11-30&Signature=1fg9hnUvEd3EOiH9KUg3P8Ydk5hAV%2
+BDkMlmVh3kL8ws%3D ####
[Sat, 28 Aug 2010 08:00:14 +0000] WARN: ##### Retry is needed, error pattern match: InternalError #####
[Sat, 28 Aug 2010 08:00:14 +0000] INFO: Closing HTTPS connection to ec2.amazonaws.com:443, reason: 'RightAws::AWSErrorHandler: error match to pattern 'InternalError''
[Sat, 28 Aug 2010 08:00:14 +0000] WARN: ##### Retry #1 is being performed. Sleeping for 0.2 sec. Whole time: 0.001593 sec ####
[Sat, 28 Aug 2010 08:00:14 +0000] INFO: Opening new HTTPS connection to ec2.amazonaws.com:443
[Sat, 28 Aug 2010 08:00:20 +0000] WARN: ##### RightAws::Ec2 returned an error: 500 Internal Server Error
<?xml version="1.0"?>
<Response><Errors><Error><Code>InternalError</Code><Message>An internal error has occurred</Message></Error></Errors><RequestID>a7a7f2b4-c6bf-4590-abcd-5322af88d801</RequestID></Response> #####
[Sat, 28 Aug 2010 08:00:20 +0000] WARN: ##### RightAws::Ec2 request:
+https://ec2.amazonaws.com:443/?AWSAccessKeyId=AKIAIVTL6LH4Q42UURGA&Action=DescribeSnapshots&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2010-08-28T08%3A00%3A08.000Z&Version=2009-11-30&Signature=1fg9hnUvEd3EOiH9KUg3P8Ydk5hAV%2
+BDkMlmVh3kL8ws%3D ####
[Sat, 28 Aug 2010 08:00:20 +0000] WARN: ##### Retry is needed, error pattern match: InternalError #####
[Sat, 28 Aug 2010 08:00:20 +0000] INFO: Closing HTTPS connection to ec2.amazonaws.com:443, reason: 'RightAws::AWSErrorHandler: error match to pattern 'InternalError''
[Sat, 28 Aug 2010 08:00:20 +0000] WARN: ##### Ooops, time is over... ####
[Sat, 28 Aug 2010 08:00:20 +0000] ERROR: #<RightAws::AwsError: InternalError: An internal error has occurred>
/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:454:in `request_info_impl'/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2.rb:139:in
+`request_info'/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:761:in `check'/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:449:in
+`request_info_impl'/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2.rb:139:in `request_info'/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:468:in
+`request_cache_or_info'/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2_ebs.rb:164:in `describe_snapshots'/var/chef/cache/cookbooks/aws/providers/ebs_volume.rb:73:in
+`class_from_file'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/provider.rb:104:in `instance_eval'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/provider.rb:104:in
+`action_prune'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:51:in `send'/usr/lib/ruby/gems/1.8/gems/chef-0.9
.8/bin/../lib/chef/runner.rb:51:in `run_action'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:109:in `converge'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:108:in
+`each'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:108:in `converge'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection.rb:94:in
+`execute_each_resource'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:116:in `call'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:116:in
+`call_iterator_block'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:85:in `step'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:104:in
+`iterate'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:55:in `each_with_index'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/
resource_collection.rb:92:in `execute_each_resource'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:82:in `converge'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/client.rb:304:in
+`converge'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/client.rb:87:in `run'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:191:in
+`run_application'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:181:in `loop'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:181:in
+`run_application'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application.rb:62:in `run'/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/chef-solo:25/usr/bin/chef-solo:19:in `load'/usr/bin/chef-solo:19.join('
')}
[Sat, 28 Aug 2010 08:00:20 +0000] ERROR: Request was:
+/?AWSAccessKeyId=AKIAIVTL6LH4Q42UURGA&Action=DescribeSnapshots&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2010-08-28T08%3A00%3A08.000Z&Version=2009-11-30&Signature=1fg9hnUvEd3EOiH9KUg3P8Ydk5hAV%2BDkMlmVh3kL8ws%3D
[Sat, 28 Aug 2010 08:00:20 +0000] ERROR: Response was: 500 -- Internal Server Error -- <?xml version="1.0"?>
<Response><Errors><Error><Code>InternalError</Code><Message>An internal error has occurred</Message></Error></Errors><RequestID>a7a7f2b4-c6bf-4590-abcd-5322af88d801</RequestID></Response>
[Sat, 28 Aug 2010 08:00:20 +0000] ERROR: aws_ebs_volume[gotime_database_master_production] (/var/chef/cache/cookbooks/database/recipes/snapshot.rb:42:in `from_file') had an error:
InternalError: An internal error has occurred
/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:454:in `request_info_impl'
/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2.rb:139:in `request_info'
/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:761:in `check'
/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:449:in `request_info_impl'
/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2.rb:139:in `request_info'
/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:468:in `request_cache_or_info'
/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2_ebs.rb:164:in `describe_snapshots'
/var/chef/cache/cookbooks/aws/providers/ebs_volume.rb:73:in `class_from_file'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/provider.rb:104:in `instance_eval'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/provider.rb:104:in `action_prune'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:51:in `send'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:51:in `run_action'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:109:in `converge'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:108:in `each'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:108:in `converge'

/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection.rb:94:in `execute_each_resource'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:116:in `call'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:116:in `call_iterator_block'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:85:in `step'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:104:in `iterate'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:55:i/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:454:in `request_info_impl': InternalError: An internal error has
+occurred (RightAws::AwsError)
        from /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2.rb:139:in `request_info'
        from /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:761:in `check'
        from /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:449:in `request_info_impl'
        from /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2.rb:139:in `request_info'             
        from /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:468:in `request_cache_or_info'
        from /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2_ebs.rb:164:in `describe_snapshots'       
        from /var/chef/cache/cookbooks/aws/providers/ebs_volume.rb:73:in `class_from_file'                   
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/provider.rb:104:in `instance_eval'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/provider.rb:104:in `action_prune' 
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:51:in `send'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:51:in `run_action'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:109:in `converge'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:108:in `each'    
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:108:in `converge'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection.rb:94:in `execute_each_resource'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:116:in `call'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:116:in `call_iterator_block'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:85:in `step'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:104:in `iterate'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:55:in `each_with_index'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection.rb:92:in `execute_each_resource'            
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:82:in `converge'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/client.rb:304:in `converge'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/client.rb:87:in `run'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:191:in `run_application'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:181:in `loop'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:181:in `run_application'
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application.rb:62:in `run'                  
        from /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/chef-solo:25
        from /usr/bin/chef-solo:19:in `load'
        from /usr/bin/chef-solo:19
n `each_with_index'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection.rb:92:in `execute_each_resource'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:82:in `converge'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/client.rb:304:in `converge'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/client.rb:87:in `run'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:191:in `run_application'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:181:in `loop'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:181:in `run_application'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application.rb:62:in `run'
/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/chef-solo:25
/usr/bin/chef-solo:19:in `load'
/usr/bin/chef-solo:19
[Sat, 28 Aug 2010 08:00:20 +0000] ERROR: Running exception handlers
[Sat, 28 Aug 2010 08:00:20 +0000] ERROR: Exception handlers complete
[Sat, 28 Aug 2010 08:00:20 +0000] ERROR: Re-raising exception: RightAws::AwsError - InternalError: An internal error has occurred
/usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:454:in `request_info_impl'
  /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2.rb:139:in `request_info'
  /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:761:in `check'
  /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:449:in `request_info_impl'
  /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2.rb:139:in `request_info'
  /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/awsbase/right_awsbase.rb:468:in `request_cache_or_info'
  /usr/lib/ruby/gems/1.8/gems/right_aws-2.0.0/lib/ec2/right_ec2_ebs.rb:164:in `describe_snapshots'
  /var/chef/cache/cookbooks/aws/providers/ebs_volume.rb:73:in `class_from_file'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/provider.rb:104:in `instance_eval'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/provider.rb:104:in `action_prune'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:51:in `send'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:51:in `run_action'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:109:in `converge'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:108:in `each'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:108:in `converge'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection.rb:94:in `execute_each_resource'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:116:in `call'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:116:in `call_iterator_block'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:85:in `step'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:104:in `iterate'

/usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection/stepable_iterator.rb:55:in `each_with_index'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/resource_collection.rb:92:in `execute_each_resource'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/runner.rb:82:in `converge'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/client.rb:304:in `converge'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/client.rb:87:in `run'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:191:in `run_application'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:181:in `loop'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application/solo.rb:181:in `run_application'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/../lib/chef/application.rb:62:in `run'
  /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/bin/chef-solo:25
  /usr/bin/chef-solo:19:in `load'
  /usr/bin/chef-solo:19