Download MacSysAdmin2016 videos

get_msa_videos.py

#!/usr/bin/env python

'''
download macsysadmin videos

requires:

pip install requests
pip install beautifulsoup4
'''

import os
import sys

import requests
from bs4 import BeautifulSoup, SoupStrainer

year = '2017'
url = 'http://docs.macsysadmin.se/%s/%sdoc.html' % (year, year)
output_dir = 'macsysadmin%s' % year

def get_videos(url):
    '''Returns a list of video links'''
    req = requests.get(url)
    soup = BeautifulSoup(req.content,
                         "html.parser",
                         parse_only=SoupStrainer('a', href=True)
                         )
    links = soup.find_all('a')
    return [link.attrs['href'] for link in links if 'mp4' in link.attrs['href']]

def save_videos(skip, urls, dir_name):
    '''Saves videos to specified directory'''
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)
    for url in urls:
        req = requests.get(url, stream=True)
        output_file = dir_name + '/' + url.split('/')[-1]
        if output_file.split('/')[-1] not in skip:
            print 'Downloading %s' % url
            with open(output_file, 'w') as f:
                for chunk in req.iter_content(chunk_size=3000):
                    f.write(chunk)
        else:
            print 'Skipping %s' % output_file

def main():
    already_downloaded = os.listdir(output_dir) if os.path.exists(output_dir) else []
    videos = get_videos(url)
    save_videos(already_downloaded, videos, output_dir)

if __name__ == '__main__':
    main()