pubmed_abstracts_downloader.py

# https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/
import requests
from selectolax.parser import HTMLParser

base_url = 'https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/'

r = requests.get(base_url)
tree = HTMLParser(r.content)

for node in tree.css('a'):
    if '.xml.gz' in node.text():
        url = base_url + node.attributes['href'] 
        print(url) 
        req = requests.get(url) 
        with open(node.text(), 'wb') as f:
            f.write(req.content)