AWS Instance attributes recursive downloader

download-instance-attributes.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Recursive AWS-style instance attributes downloader."""

import logging
import os
import re

import requests

log_formatter = logging.Formatter('%(asctime)s, %(levelname)s %(message)s')

logging.basicConfig(level=logging.INFO, formatter=log_formatter)
logger = logging.getLogger(__name__)

seed_url = 'http://169.254.169.254/latest'

def recurse(url):
    logger.info('retrieving url=%s', url)
    resp = requests.get(url)
    if resp.status_code / 100 != 2:
        logger.debug('url=%s returned non-2xx status code=%s (skipping)', url, resp.status_code)
        return []

    attr = re.sub(r'''http:\/\/[^\/]+\/''', '', url)

    if not os.path.exists(attr):
        os.makedirs(attr)
    with open('%s.txt' % (attr,), 'w') as fh:
        fh.write(resp.text)

    new_urls = ['%s/%s' % (url, attr) for attr in resp.text.split('\n')] if resp.text else []
    return new_urls

if __name__ == '__main__':
    urls = [seed_url]
    while len(urls):
        urls += recurse(urls.pop())