Extract MITRE ATT&CK techniques into a file.

mitre_extract.py

"""Extract MITRE ATT&CK techniques into a file."""
import bs4 as bs
import requests

root_url = "https://attack.mitre.org"
file_name = "mitre.txt"


def get_urls():
    """Get MITRE ATT&CK URLs for processing."""
    urls = list()
    url = "https://attack.mitre.org/techniques/enterprise/"
    response = requests.get(url)
    soup = bs.BeautifulSoup(response.content,'html.parser')
    rows = soup.find_all('tr', {'class': 'technique'})
    for row in rows:
        select = row.select_one(":nth-child(2)")
        full_url = root_url + select.find('a')['href']
        urls.append(full_url)
        # print(full_url)
    return urls
    
def derive_id(url):
    """Use the URL to derive the technique ID."""
    parts = url.split('/')
    save = None
    for idx, item in enumerate(parts):
        if item != 'techniques':
            continue
        save = parts[idx+1:]
    return '.'.join([x for x in save if x != ''])
    
def extract_content(url):
    """Extract content from the HTML to form our structure."""
    # print("PROCESSING:", url)
    tmp = {'url': url, 'id': derive_id(url), 'label': None, 'description': None, 'examples': list()}
    response = requests.get(url)
    soup = bs.BeautifulSoup(response.content,'html.parser')
    container = soup.find('div', {'class':'container-fluid'})

    label = container.find('h1').text.replace('\n', '').replace('  ', '')
    tmp['label'] = label
    content = container.find('div', {'class': 'description-body'})
    description = ' '.join([x.text for x in content.find_all('p')])
    tmp['description'] = description
    content = soup.find('h2', {'id': 'examples'})
    if not content:
        return tmp
    content = content.findNext('table')
    examples = list()
    rows = content.find_all('tr')
    for row in rows[1:]:
       select = row.select_one(":nth-child(3)").find('p')
       examples.append(select.text)
    tmp['examples'] = examples
    return tmp

if __name__ == "__main__":
    """I process, therefore I am."""
    mapping = list()
    urls = get_urls()
    f = open(file_name, "w")
    for url in urls:
        item = extract_content(url)
        f.write("Technique:\n %s (%s)\n" % (item['label'], item['id']))
        f.write("\n")
        f.write("Description:\n %s\n" % item['description'])
        f.write("\n")
        f.write("Examples: \n%s\n" % '\n'.join(item['examples']))
        f.write("\n")
    f.close()