Skip to content

Instantly share code, notes, and snippets.

@abs51295
Created October 30, 2018 13:46
Show Gist options
  • Save abs51295/4c44ce180372c60271b3568929fd6eb7 to your computer and use it in GitHub Desktop.
Save abs51295/4c44ce180372c60271b3568929fd6eb7 to your computer and use it in GitHub Desktop.
Python package collection
from pip._internal.req.req_file import parse_requirements
from pip._internal.download import PipSession
from pip._vendor.distlib.util import normalize_name
import requests
import requests_cache
from datetime import timedelta
import json
## To cache responses of requests objects, this can make things faster.
requests_cache.install_cache(expire_after=timedelta(hours=5))
manifest_json = [{
"ecosystem": "pypi",
"package_list": []
}]
session = PipSession()
with open("python-bigquery-data.json", "r") as f, open('error-log-pip.txt', 'a') as log:
content = json.load(f)
for x in content:
if x.get('content'):
with open("temp-requirements.txt", "w") as w:
w.write(x.get('content'))
req_names = []
try:
# This parses the valid requirements files.
for p in parse_requirements("temp-requirements.txt", session=session):
if p.name:
name = normalize_name(p.name)
# A sanity check to see if the package actually exists on pypi.
r = requests.get(url='https://pypi.org/pypi/{}/json'.format(name), headers={"Accept": "application/json"})
if r.status_code == 200:
print("Package: {} done".format(name))
req_names.append(name)
except Exception as e:
log.write(e)
manifest_json[0].get("package_list").append(req_names)
# Now you have everything you need in manifest_json object. Dump it in a file or use it anywhere.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment