Created
October 5, 2015 17:34
-
-
Save pyr-revs/d768984ed68500bdbeb9 to your computer and use it in GitHub Desktop.
Launch xgboost via AWS Lambda
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
console.log('Loading function'); | |
var ec2Region = 'ap-northeast-1'; | |
var s3Region = ec2Region; | |
var snsRegion = ec2Region; | |
var s3Bucket = 'mybucket'; | |
var shellScriptS3Key = 'sh/launch_xgboost.sh'; | |
var shellScriptS3Path = 's3://' + s3Bucket + '/' + shellScriptS3Key; | |
var iamInstanceProfile = 'my_ec2_role'; | |
var availabilityZone = ec2Region + 'a'; | |
var spotPrice = '0.1'; | |
var imageId = 'ami-9a2fb89a'; | |
var instanceType = 'c3.2xlarge'; | |
var securityGroup = 'launch-wizard-1'; | |
var keyName = 'my_ssh_keypair'; | |
var userData = (function () {/*#!/bin/bash | |
tmp=/root/sudoers_tmp | |
cat /etc/sudoers > $tmp | |
cat >> $tmp <<EOF | |
Defaults:ec2-user !requiretty | |
EOF | |
cat $tmp > /etc/sudoers | |
yum -y update | |
yum groupinstall -y "Development tools" | |
yum -y install gcc-c++ python27-devel atlas-sse3-devel lapack-devel | |
pip install numpy | |
pip install scipy | |
pip install pandas | |
aws s3 cp %s /home/ec2-user/launch_xgboost.sh | |
chown ec2-user /home/ec2-user/launch_xgboost.sh | |
chmod +x /home/ec2-user/launch_xgboost.sh | |
su - ec2-user /home/ec2-user/launch_xgboost.sh | |
*/}).toString().match(/[^]*\/\*([^]*)\*\/\}$/)[1]; | |
var shellScriptContents = (function () {/*#!/bin/bash | |
git clone https://github.com/dmlc/xgboost.git | |
cd xgboost | |
./build.sh > build.log 2>&1 | |
cd python-package | |
sudo -s python setup.py install > setup.log 2>&1 | |
export AWS_DEFAULT_REGION=%s | |
aws sns publish --topic-arn arn:aws:sns:ap-northeast-1:xxxxxxxxxxxx:My-Sns-Topic --subject "Launch xgboost Done" --message "Launch xgboost Done!!" | |
*/}).toString().match(/[^]*\/\*([^]*)\*\/\}$/)[1]; | |
exports.handler = function(event, context) { | |
var util = require('util'); | |
var AWS = require('aws-sdk'); | |
// Write sh file for xgboost launch to S3 | |
AWS.config.region = s3Region; | |
var shellScriptContentsFormatted = util.format(shellScriptContents, snsRegion); | |
var s3 = new AWS.S3(); | |
var s3Params = {Bucket: s3Bucket, Key: shellScriptS3Key, Body: shellScriptContentsFormatted}; | |
var s3Options = {partSize: 10 * 1024 * 1024, queueSize: 1}; | |
//console.log(shellScriptContentsFormatted); | |
s3.upload(s3Params, s3Options, function(err, data) { | |
if (err) { | |
console.log(err, err.stack); | |
context.fail('[Fail]'); | |
} | |
else { | |
console.log(data); | |
// Lauch EC2 Spot Instance with UserData | |
var userDataFormatted = util.format(userData, shellScriptS3Path); | |
var userDataBase64 = new Buffer(userDataFormatted).toString('base64'); | |
var ec2LaunchParams = { | |
SpotPrice: spotPrice, | |
LaunchSpecification : { | |
IamInstanceProfile: { | |
Name: iamInstanceProfile | |
}, | |
ImageId: imageId, | |
InstanceType: instanceType, | |
KeyName: keyName, | |
Placement: { | |
AvailabilityZone: availabilityZone | |
}, | |
SecurityGroups: [ | |
securityGroup | |
], | |
UserData: userDataBase64 | |
} | |
}; | |
//console.log(params); | |
AWS.config.region = ec2Region; | |
var ec2 = new AWS.EC2(); | |
ec2.requestSpotInstances(ec2LaunchParams, function(err, data) { | |
if (err) { | |
console.log(err, err.stack); | |
context.fail('[Fail]'); | |
} | |
else { | |
console.log(data); | |
context.succeed('[Succeed]'); | |
} | |
}); | |
} | |
}); | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment