Pallet + EC2 == too hard :-(

core.clj

(ns staging.core
  (:require 
    [pallet.core :as core]
    [pallet.compute :as compute]
    [pallet.phase :as phase]
    [pallet.crate.automated-admin-user :as automated-admin-user]
    [staging.nodes :as staging-nodes]
    [staging.crates.mongodb :as mongodb]))


(def ec2-service
  (compute/compute-service-from-config-file :ec2))


; FIXME
;
; Does not work.  Need a way to specify existing SSH keypair before launch,
; preferably at the cluster-spec level.
; 
; http://code.google.com/p/jclouds/issues/detail?id=602
;
;(def ubuntu-admin-user
;  (core/admin-user "ubuntu"
;    :private-key-path "~/.ssh/assistly/staging.pem"))


; FIXME
;
; I want to ensure the "staging" security group is created before any
; instance is launched and have all instances "inherit" that security group.
;
; If all instances are destroyed I want to destroy the "staging" security
; group.
;
; I want to automatically create security groups at the server-spec level
; and have all instances "inherit" those security groups at the group-spec
; level.
;
; Eg, if a group-spec extends "with-redis" and "with-mongodb" I want to make
; sure any instance launched from that group-spec is in both the "mongodb"
; and "redis" security groups.  If all instances in the "redis" security
; group are destroyed I want to destroy the "redis" security group.
;
; I also want to be able to specify security groups as members of other
; security groups.  Eg, so that only instances in the "web" security group
; are able to talk to instances in the "database" security group.
;
; I have no idea where to call these... 
;
; (.createSecurityGroupInRegion ec2-security-group-client "us-east-1" "sg" "desc")
; (.deleteSecurityGroupInRegion ec2-security-group-client "us-east-1" "sg")
;
(def ec2-security-group-client
  (..
    (clojure.contrib.reflect/get-field
      org.jclouds.ec2.compute.EC2ComputeService "ec2Client" (.compute ec2-service))
    getSecurityGroupServices))


; With an automatically-created admin user account for the current user
;
; FIXME
;
; Get rid of this as soon as this bug is fixed:
; http://code.google.com/p/jclouds/issues/detail?id=602
;
(def with-automated-admin-user
  (core/server-spec
    :phases {:bootstrap (phase/phase-fn (automated-admin-user/automated-admin-user))}))


; With MongoDB from the 10gen repository
;
; We use the 10gen-provided MongoDB to get the most recent packaged version.
; 
(def with-mongodb-10gen
  (core/server-spec
   :phases {:configure (phase/phase-fn (mongodb/install))
            :start-mongodb (phase/phase-fn (mongodb/start))
            :stop-mongodb (phase/phase-fn (mongodb/stop))
            :restart-mongodb (phase/phase-fn (mongodb/restart))}))


; Recommended EC2 configuration:
; http://www.mongodb.org/display/DOCS/Amazon+EC2
; "use a 64 bit instance as this is required for all MongoDB databases of
; significant size"
;
(def mongodb-us-east-1a
  (core/group-spec
    "mongodb"
    :extends [with-automated-admin-user
              with-mongodb-10gen]
    :node-spec staging-nodes/ubuntu-1104-m1-large-us-east-1a))


; Recommended EC2 configuration:
; http://www.mongodb.org/display/DOCS/Amazon+EC2
; "use a 64 bit instance as this is required for all MongoDB databases of
; significant size"
;
(def mongodb-us-east-1b
  (core/group-spec
    "mongodb"
    :extends [with-automated-admin-user
              with-mongodb-10gen]
    :node-spec staging-nodes/ubuntu-1104-m1-large-us-east-1b))


(def mongodb-cluster
  (core/cluster-spec
    "mongodb"
    :groups [mongodb-us-east-1a
             mongodb-us-east-1b]))

crates.mongodb.clj

(ns staging.crate.mongodb
  (:require
    [pallet.action.exec-script :as exec-script]
    [pallet.action.directory :as directory]
    [pallet.action.file :as file]
    [pallet.action.package :as package]
    [pallet.action.service :as service]
    [pallet.script :as script]))


(def default-configuration
  {:dbpath "/var/lib/mongodb"
   :logpath "/var/log/mongodb/mongodb.log"
   :logappend true
   :port 27017
   :cpu false
   :noauth true
   :auth false
   :verbose false
   :objcheck false
   :quota false
   :diaglog 0
   :nocursors false
   :nohints false
   :nohttpinterface true
   :noscripting false
   :notablescan false
   :noprealloc false
   :master false
   :slave false
   :journal true})


(defn install
  "Install MongoDB server (from the 10gen repository)" 
  [session] 
  (-> session 
    (package/package-source 
      "10gen" 
      :aptitude {:source-type "deb"
                 :url "http://downloads-distro.mongodb.org/repo/ubuntu-upstart"
                 :release "dist"
                 :scopes ["10gen"]
                 :key-id "7F0CEB10"} )
    (package/package-manager :update) 
    (package/package "mongodb-10gen" :action :install)
    (service/service "mongodb" :action :restart)))


(defn start
  "Start MongoDB service"
  [session]
  (-> session
    (service/service "mongodb" :action :start)))


(defn stop
  "Stop MongoDB service"
  [session]
  (-> session
    (service/service "mongodb" :action :stop)))


(defn restart
  "Restart MongoDB service"
  [session]
  (-> session
    (service/service "mongodb" :action :restart)))

Launching cluster fails

$ lein pallet converge staging.core/mongodb-cluster 1

WARN  implementation - pallet.compute.vmfest provider failed to load: java.io.FileNotFoundException: Could not locate vmfest/virtualbox/virtualbox__init.class or vmfest/virtualbox/virtualbox.clj on classpath:  (vmfest.clj:1)
INFO  core - retrieving nodes
ERROR main - Exception
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.NullPointerException
	at clojure.lang.LazySeq.sval(LazySeq.java:47)
	at clojure.lang.LazySeq.seq(LazySeq.java:56)
	at clojure.lang.RT.seq(RT.java:450)
	at clojure.core$seq.invoke(core.clj:122)
	at clojure.core$r.invoke(core.clj:793)
	at pallet.core$nodes_in_set.invoke(core.clj:841)
	at pallet.core$session_with_groups.invoke(core.clj:915)
	at pallet.core$converge_STAR_.invoke(core.clj:980)
	at pallet.core$converge.doInvoke(core.clj:1148)
	at clojure.lang.RestFn.applyTo(RestFn.java:139)
	at clojure.core$apply.invoke(core.clj:540)
	at pallet.task.converge$converge.doInvoke(converge.clj:40)
	at clojure.lang.RestFn.invoke(RestFn.java:439)
	at clojure.lang.Var.invoke(Var.java:373)
	at clojure.lang.AFn.applyToHelper(AFn.java:167)
	at clojure.lang.Var.applyTo(Var.java:482)
	at clojure.core$apply.invoke(core.clj:542)
	at pallet.main_invoker$invoke.invoke(main_invoker.clj:102)
	at clojure.lang.Var.invoke(Var.java:373)
	at pallet.main$pallet_task.invoke(main.clj:138)
	at pallet.main$_main.doInvoke(main.clj:152)
	at clojure.lang.RestFn.invoke(RestFn.java:482)
	at clojure.lang.Var.invoke(Var.java:381)
	at user$eval368.invoke(NO_SOURCE_FILE:1)
	at clojure.lang.Compiler.eval(Compiler.java:5424)
	at clojure.lang.Compiler.eval(Compiler.java:5415)
	at clojure.lang.Compiler.eval(Compiler.java:5415)
	at clojure.lang.Compiler.eval(Compiler.java:5391)
	at clojure.core$eval.invoke(core.clj:2382)
	at clojure.main$eval_opt.invoke(main.clj:235)
	at clojure.main$initialize.invoke(main.clj:254)
	at clojure.main$null_opt.invoke(main.clj:279)
	at clojure.main$main.doInvoke(main.clj:354)
	at clojure.lang.RestFn.invoke(RestFn.java:421)
	at clojure.lang.Var.invoke(Var.java:369)
	at clojure.lang.AFn.applyToHelper(AFn.java:163)
	at clojure.lang.Var.applyTo(Var.java:482)
	at clojure.main.main(main.java:37)
Caused by: java.lang.RuntimeException: java.lang.NullPointerException
	at clojure.lang.LazySeq.sval(LazySeq.java:47)
	at clojure.lang.LazySeq.seq(LazySeq.java:56)
	at clojure.lang.RT.seq(RT.java:450)
	at clojure.core$seq.invoke(core.clj:122)
	at clojure.core$zipmap.invoke(core.clj:2301)
	at pallet.core$node_map_with_prefix.invoke(core.clj:766)
	at pallet.core$nodes_in_set.invoke(core.clj:835)
	at pallet.core$nodes_in_set$fn__6162.invoke(core.clj:841)
	at clojure.core$map$fn__3699.invoke(core.clj:2096)
	at clojure.lang.LazySeq.sval(LazySeq.java:42)
	... 37 more
Caused by: java.lang.NullPointerException
	at clojure.core$name.invoke(core.clj:1245)
	at pallet.core$group_with_prefix$fn__6129.invoke(core.clj:761)
	at clojure.lang.AFn.applyToHelper(AFn.java:161)
	at clojure.lang.AFn.applyTo(AFn.java:151)
	at clojure.core$apply.invoke(core.clj:542)
	at clojure.core$update_in.doInvoke(core.clj:4969)
	at clojure.lang.RestFn.invoke(RestFn.java:445)
	at pallet.core$group_with_prefix.invoke(core.clj:760)
	at pallet.core$node_map_with_prefix$fn__6133.invoke(core.clj:765)
	at clojure.core$map$fn__3699.invoke(core.clj:2096)
	at clojure.lang.LazySeq.sval(LazySeq.java:42)
	... 46 more
java.lang.RuntimeException: java.lang.NullPointerException

Launching us-east-1a instance fails

$ lein pallet converge staging.core/mongodb-us-east-1a 1
WARN  implementation - pallet.compute.vmfest provider failed to load: java.io.FileNotFoundException: Could not locate vmfest/virtualbox/virtualbox__init.class or vmfest/virtualbox/virtualbox.clj on classpath:  (vmfest.clj:1)
INFO  core - retrieving nodes
INFO  core - converging nodes
INFO  core - adjust-node-count :mongodb 1
INFO  core - Starting 1 nodes for :mongodb os-family :ubuntu
INFO  jclouds - building node template for :mongodb
INFO  jclouds -   authorizing /Users/david/.ssh/id_rsa.pub
INFO  jclouds -   options {:image-id "us-east-1/ami-1aad5273", :os-64-bit true, :os-description-matches "11.04", :os-family :ubuntu, :hardware-id "m1.large", :location-id "us-east-1a", :inbound-ports [22]}
WARN  compute - to avoid creating temporary keys in aws-ec2, use templateOption overrideLoginCredentialWith(id_rsa)
WARN  compute - to avoid creating temporary keys in aws-ec2, use templateOption overrideLoginCredentialWith(id_rsa)
ERROR futures - Adjust node count exception: requested location us-east-1a, which is not in the configured locations: {ap-northeast-1=https://ec2.ap-northeast-1.amazonaws.com, eu-west-1=https://ec2.eu-west-1.amazonaws.com, us-east-1=https://ec2.us-east-1.amazonaws.com, us-west-1=https://ec2.us-west-1.amazonaws.com, ap-southeast-1=https://ec2.ap-southeast-1.amazonaws.com}
java.lang.IllegalArgumentException: requested location us-east-1a, which is not in the configured locations: {ap-northeast-1=https://ec2.ap-northeast-1.amazonaws.com, eu-west-1=https://ec2.eu-west-1.amazonaws.com, us-east-1=https://ec2.us-east-1.amazonaws.com, us-west-1=https://ec2.us-west-1.amazonaws.com, ap-southeast-1=https://ec2.ap-southeast-1.amazonaws.com}
	at com.google.common.base.Preconditions.checkArgument(Preconditions.java:115)
	at org.jclouds.location.functions.RegionToEndpointOrProviderIfNull.apply(RegionToEndpointOrProviderIfNull.java:63)
	at org.jclouds.location.functions.RegionToEndpointOrProviderIfNull.apply(RegionToEndpointOrProviderIfNull.java:43)
	at org.jclouds.rest.internal.RestAnnotationProcessor.getEndpointInParametersOrNull(RestAnnotationProcessor.java:670)
	at org.jclouds.rest.internal.RestAnnotationProcessor.getEndpointFor(RestAnnotationProcessor.java:702)
	at org.jclouds.rest.internal.RestAnnotationProcessor.createRequest(RestAnnotationProcessor.java:372)
	at org.jclouds.rest.internal.AsyncRestClientProxy.createListenableFuture(AsyncRestClientProxy.java:145)
	at org.jclouds.rest.internal.AsyncRestClientProxy.invoke(AsyncRestClientProxy.java:126)
	at $Proxy71.runInstancesInRegion(Unknown Source)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:597)
	at org.jclouds.concurrent.internal.SyncProxy.invoke(SyncProxy.java:130)
	at $Proxy72.runInstancesInRegion(Unknown Source)
	at org.jclouds.ec2.compute.strategy.EC2CreateNodesInGroupThenAddToSet.createNodesInRegionAndZone(EC2CreateNodesInGroupThenAddToSet.java:166)
	at org.jclouds.aws.ec2.compute.strategy.AWSEC2CreateNodesInGroupThenAddToSet.createNodesInRegionAndZone(AWSEC2CreateNodesInGroupThenAddToSet.java:99)
	at org.jclouds.ec2.compute.strategy.EC2CreateNodesInGroupThenAddToSet.createKeyPairAndSecurityGroupsAsNeededThenRunInstances(EC2CreateNodesInGroupThenAddToSet.java:152)
	at org.jclouds.ec2.compute.strategy.EC2CreateNodesInGroupThenAddToSet.execute(EC2CreateNodesInGroupThenAddToSet.java:116)
	at org.jclouds.compute.internal.BaseComputeService.createNodesInGroup(BaseComputeService.java:229)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:597)
	at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:90)
	at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
	at org.jclouds.compute$create_nodes.invoke(compute.clj:213)
	at clojure.lang.AFn.applyToHelper(AFn.java:172)
	at clojure.lang.AFn.applyTo(AFn.java:151)
	at clojure.core$apply.invoke(core.clj:540)
	at org.jclouds.compute$run_nodes.doInvoke(compute.clj:215)
	at clojure.lang.RestFn.invoke(RestFn.java:457)
	at pallet.compute.jclouds.JcloudsService.run_nodes(jclouds.clj:404)
	at pallet.core$create_nodes.invoke(core.clj:408)
	at pallet.core$adjust_node_count.invoke(core.clj:450)
	at pallet.core$parallel_adjust_node_counts$p_a_n_c_future__5973$fn__5974.invoke(core.clj:473)
	at clojure.lang.AFn.call(AFn.java:18)
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
	at java.util.concurrent.FutureTask.run(FutureTask.java:138)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
INFO  core - lift-nodes phases [:settings :configure], groups [:mongodb]
INFO  core - apply-phase :pallet.phase/pre-settings for :mongodb with 0 nodes
INFO  core - apply-phase :settings for :mongodb with 0 nodes
INFO  core - apply-phase :pallet.phase/post-settings for :mongodb with 0 nodes
INFO  core - apply-phase :pallet.phase/pre-configure for :mongodb with 0 nodes
INFO  core - apply-phase :configure for :mongodb with 0 nodes
INFO  core - apply-phase :pallet.phase/post-configure for :mongodb with 0 nodes

nodes.clj

(ns staging.nodes
  (:require 
    [pallet.core :as core]))


; Defines an EC2 m1.large instance in us-east-1a using the Ubuntu
; 11.04 AMI
;
(def ubuntu-1104-m1-large-us-east-1a
  (core/node-spec
   :image {:os-family :ubuntu
           :os-description-matches "11.04"
           :os-64-bit true
           :image-id "us-east-1/ami-1aad5273"}
   :hardware {:hardware-id "m1.large"}
   :network {:inbound-ports [22]
             ; FIXME
             ; Does not work.
             ; "requested location us-east-1a, which is not in the configured locations"
             :location-id "us-east-1b"
             ; FIXME
             ; Need to make sure these security groups are pre-created
             ;:security-groups ["staging"]
             ; FIXME
             ; Looks like this is broken...
             ; http://code.google.com/p/jclouds/issues/detail?id=602
             ;:key-pair "us-east-1a_staging"
             }))


; Defines an EC2 m1.large instance in us-east-1b using the Ubuntu
; 11.04 AMI
;
(def ubuntu-1104-m1-large-us-east-1b
  (core/node-spec
   :image {:os-family :ubuntu
           :os-description-matches "11.04"
           :os-64-bit true
           :image-id "us-east-1/ami-1aad5273"}
   :hardware {:hardware-id "m1.large"}
   :network {:inbound-ports [22]
             ; FIXME
             ; Does not work.
             ; "requested location us-east-1a, which is not in the configured locations"
             :location-id "us-east-1b"
             ; FIXME
             ; Need to make sure these security groups are pre-created
             ;:security-groups ["staging"]
             ; FIXME
             ; Looks like this is broken...
             ; http://code.google.com/p/jclouds/issues/detail?id=602
             ;:key-pair "us-east-1a_staging"
             }))

Availability zone issue is logged here:
http://code.google.com/p/jclouds/issues/detail?id=608

Adrian has been unable to reproduce the issue, so needs help to get this resolved for you

I spoke too soon - reproduced using the clojure api