Dataduct AMI

data-pipeline-ami.json

{
  "variables": {
    "version": "2015-11-08"
  },
  "builders": [
    {
      "type": "amazon-ebs",
      "name": "amazon-ebs",
      "region": "us-east-1",
      "source_ami": "ami-ee146b84",
      "instance_type": "c3.large",
      "ssh_username": "ubuntu",
      "ami_name": "mango-data-pipeline-ebs-{{user `version`}}",
      "force_deregister": true
    },
    {
      "type": "amazon-ebs",
      "name": "amazon-ebs-hvm",
      "region": "us-east-1",
      "source_ami": "ami-f7136c9d",
      "instance_type": "c3.large",
      "ssh_username": "ubuntu",
      "ami_name": "mango-data-pipeline-ebs-hvm-{{user `version`}}",
      "force_deregister": true,
      "enhanced_networking": true
    }
  ],
  "provisioners": [
    {
      "type": "file",
      "source": "data-pipeline/dataduct.cfg",
      "destination": "/tmp/dataduct.cfg"
    },
    {
      "type": "shell",
      "execute_command": "{{ .Vars }} sudo -E -S sh '{{ .Path }}'",
      "inline": [
        "apt-get update",
        "DEBIAN_FRONTEND=noninteractive apt-get install -y unzip openjdk-7-jre libmysqlclient-dev libpq-dev python-dev python-pip",
        "pip install dataduct",
        "adduser --disabled-password ec2-user",
        "echo 'ec2-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ec2-user",
        "mkdir /home/ec2-user/.dataduct",
        "mv /tmp/dataduct.cfg /home/ec2-user/.dataduct/",
        "chown -R ec2-user:ec2-user /home/ec2-user/.dataduct"
      ]
    }
  ]
}

dataduct.cfg

etl:
    S3_ETL_BUCKET: your-etl-bucket
    S3_BASE_PATH: your-base-path

logging:
    LOG_DIR: ~/.dataduct