Python package collection

python-data-collection.py

from pip._internal.req.req_file import parse_requirements
from pip._internal.download import PipSession
from pip._vendor.distlib.util import normalize_name
import requests
import requests_cache
from datetime import timedelta
import json

## To cache responses of requests objects, this can make things faster.
requests_cache.install_cache(expire_after=timedelta(hours=5))

manifest_json = [{
    "ecosystem": "pypi",
    "package_list": []
}]
session = PipSession()

with open("python-bigquery-data.json", "r") as f, open('error-log-pip.txt', 'a') as log:
    content = json.load(f)
    for x in content:
        if x.get('content'):
            with open("temp-requirements.txt", "w") as w:
                w.write(x.get('content'))
            req_names = []
            try:
                # This parses the valid requirements files.
                for p in parse_requirements("temp-requirements.txt", session=session):
                    if p.name:
                        name = normalize_name(p.name)
                        # A sanity check to see if the package actually exists on pypi.
                        r = requests.get(url='https://pypi.org/pypi/{}/json'.format(name), headers={"Accept": "application/json"})
                        if r.status_code == 200:
                            print("Package: {} done".format(name))
                            req_names.append(name)
            except Exception as e:
                log.write(e)
            manifest_json[0].get("package_list").append(req_names)
            
# Now you have everything you need in manifest_json object. Dump it in a file or use it anywhere.