Amazon CloudWatch + wget + AWS Lambda + DynamoDB + S3

readme.md

• wget + AWS Lambda + DynamoDB + S3

callShellDelete.sh

#!/bin/sh
ls -la /tmp/

rm -rf /tmp/wget.$1
rm /tmp/urls.txt.$1
rm /tmp/$1.tar.gz

ls -la /tmp/

callShellWget.sh

#!/bin/sh
bin/wget -P /tmp/wget.$1 -i /tmp/urls.txt.$1 -r -l 1 -nv

tar cvzf /tmp/$1.tar.gz /tmp/wget.$1

index.js

'use strict';

const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const documentClient = new AWS.DynamoDB.DocumentClient({ region: 'ap-northeast-1' });
const execSync = require('child_process').execSync;
const fs = require('fs');

function getDb(domainKey) {

    var params = {
        TableName: process.env.dynamoDb_table,
        Key: {
            'domain': domainKey
        }
    };

    return documentClient.get(params).promise();
}

function createUrlFile(urls, targetDate) {

    var urlString = urls.join('\n');
    console.log("urls:", urlString);

    return new Promise(function(resolve, reject) {
        fs.writeFileSync("/tmp/urls.txt." + targetDate, urlString);
        resolve("createFile end.");
    });
}

function callShellScript(cmd) {

    return new Promise(function(resolve, reject) {
        const childLog = execSync(cmd, (error) => {
            // reject(error);
        }).toString();

        console.log("callShellScript: ", childLog);
        resolve("callShellScript: success");
    });
}

function s3Upload(domainKey, targetDate) {

    var params = {
        Bucket: process.env.s3_bucket,
        Key: domainKey + "/" + targetDate.substr(0, 6) + "/" + targetDate + ".tar.gz",
        ContentType: "application/tar+gzip",
        Body: fs.readFileSync("/tmp/" + targetDate + ".tar.gz"),
    };

    return new Promise(function(resolve, reject) {
        s3.putObject(params, (error, data) => {
            if (error) {
                console.log("s3Upload:", error);
                reject(error);
            }
            else {
                console.log("s3Upload:", "success");
                resolve("s3Upload: success");
            }
        });
    });
}

exports.handler = (event, context, callback) => {

    var dtYmdHis = new Date().toISOString().replace(/[-:]/g, '').replace(/T/, '-').replace(/\..+/, '');
    console.log("dtYmdHis:", dtYmdHis);

    getDb(event.domain)
        .then((r) => {
            console.log('getDb result:', r);
            createUrlFile(r.Item.urls, dtYmdHis);
        })
        .then((r) => {
            callShellScript("/bin/bash ./callShellWget.sh " + dtYmdHis);
        })
        .then((r) => {
            s3Upload(event.domain, dtYmdHis);
        })
        .then((r) => {
            callShellScript("/bin/bash ./callShellDelete.sh " + dtYmdHis);
        })
        .then((r) => {
            callback(null, r);
        })
        .catch((e) => {
            console.log('error:', e);
        });
};