Launch xgboost via AWS Lambda

launch-xgboost_via_aws_lambda.js

console.log('Loading function');

var ec2Region = 'ap-northeast-1';
var s3Region = ec2Region;
var snsRegion = ec2Region;

var s3Bucket = 'mybucket';
var shellScriptS3Key = 'sh/launch_xgboost.sh';
var shellScriptS3Path = 's3://' + s3Bucket + '/' + shellScriptS3Key;

var iamInstanceProfile = 'my_ec2_role';

var availabilityZone = ec2Region + 'a';
var spotPrice = '0.1';
var imageId = 'ami-9a2fb89a';
var instanceType = 'c3.2xlarge';
var securityGroup = 'launch-wizard-1';
var keyName = 'my_ssh_keypair';

var userData = (function () {/*#!/bin/bash
tmp=/root/sudoers_tmp
cat /etc/sudoers > $tmp
cat >> $tmp <<EOF
Defaults:ec2-user !requiretty
EOF
cat $tmp > /etc/sudoers
yum -y update
yum groupinstall -y "Development tools"
yum -y install gcc-c++ python27-devel atlas-sse3-devel lapack-devel
pip install numpy
pip install scipy
pip install pandas
aws s3 cp %s /home/ec2-user/launch_xgboost.sh
chown ec2-user /home/ec2-user/launch_xgboost.sh
chmod +x /home/ec2-user/launch_xgboost.sh
su - ec2-user /home/ec2-user/launch_xgboost.sh
*/}).toString().match(/[^]*\/\*([^]*)\*\/\}$/)[1];

var shellScriptContents = (function () {/*#!/bin/bash
git clone https://github.com/dmlc/xgboost.git
cd xgboost
./build.sh > build.log 2>&1
cd python-package
sudo -s python setup.py install > setup.log 2>&1
export AWS_DEFAULT_REGION=%s
aws sns publish --topic-arn arn:aws:sns:ap-northeast-1:xxxxxxxxxxxx:My-Sns-Topic --subject "Launch xgboost Done" --message "Launch xgboost Done!!"
*/}).toString().match(/[^]*\/\*([^]*)\*\/\}$/)[1];

exports.handler = function(event, context) {
    var util = require('util');
    var AWS = require('aws-sdk');

    // Write sh file for xgboost launch to S3
    AWS.config.region = s3Region;
    var shellScriptContentsFormatted = util.format(shellScriptContents, snsRegion);
    var s3 = new AWS.S3();
    var s3Params = {Bucket: s3Bucket, Key: shellScriptS3Key, Body: shellScriptContentsFormatted};
    var s3Options = {partSize: 10 * 1024 * 1024, queueSize: 1};
    //console.log(shellScriptContentsFormatted);

    s3.upload(s3Params, s3Options, function(err, data) {
        if (err) {
            console.log(err, err.stack);
            context.fail('[Fail]');
        }
        else {
            console.log(data);

            // Lauch EC2 Spot Instance with UserData
            var userDataFormatted = util.format(userData, shellScriptS3Path);
            var userDataBase64 = new Buffer(userDataFormatted).toString('base64');

            var ec2LaunchParams = {
                SpotPrice: spotPrice, 
                LaunchSpecification : {
                    IamInstanceProfile: {
                      Name: iamInstanceProfile
                    },
                    ImageId: imageId,
                    InstanceType: instanceType,
                    KeyName: keyName,
                    Placement: {
                      AvailabilityZone: availabilityZone
                    },
                    SecurityGroups: [
                        securityGroup
                    ],
                    UserData: userDataBase64
                }
            };
            //console.log(params);

            AWS.config.region = ec2Region;
            var ec2 = new AWS.EC2();
            ec2.requestSpotInstances(ec2LaunchParams, function(err, data) {
                if (err) {
                    console.log(err, err.stack);
                    context.fail('[Fail]');
                }
                else {
                    console.log(data);
                    context.succeed('[Succeed]');
                }
            });
        }
    });
};