This snippet is used to retrieve data from https://heasarc.gsfc.nasa.gov/FTP/

Custom API.py

!pip install validators

from bs4 import BeautifulSoup
import requests
import validators

def get_data(url):
    soup = BeautifulSoup(requests.get(url).text)
    url_data = []

    for a in soup.find_all("a"): # retrieve all hyperlink tag 
        if a.has_attr('href'): # check there is any href attribute or not
            url2 = a["href"] # get the hyperlink
            data = {}
            
            if validators.url(url2): # check it is a valid url or sub directory or file
                data['label'] = "url" 
            else :
                data['label'] = a.text
                url2 = url+url2 # if sub directory, make a valid url    
                    
            data['url'] = url2
            

            url_data.append(data)
            
    print(url_data)