collect.py against distlib data

collect.py

# find . -name project.json | python3 collect.py
#
# total_projects:    41228
# total_projects_eq: 182
# % affected:        0.44%
# total_files:       285248
# total_files_eq:    1276
# % affected:        0.45%
# total_reqs:        642447
# total_reqs_bare:   460080
# % affected:        71.61%

import sys
import json

total_projects = 0
total_projects_eq = 0
total_files = 0
total_files_eq = 0
total_reqs = 0
total_reqs_bare = 0

for line in sys.stdin.readlines():
    with open(line.strip(), 'rU') as fp:
        project = json.loads(fp.read())

    try:
        project_files = project['files']
        assert project_files
    except (KeyError, AssertionError):
        continue

    project_eq = False
    total_files += len(project_files)
    total_projects += 1

    for entry in project_files:
        try:
            reqs = set(r for rr in entry['requirements']['run_requires'] for r in rr.get('requires', []))
        except AttributeError:
            reqs = set(r for r in entry['requirements']['run_requires'])
        except KeyError:
            continue

        file_eq = False
        for req in reqs:
            total_reqs += 1
            if '(' not in req:
                total_reqs_bare += 1
            if '==' in req:
                file_eq = True
                project_eq = True

        if file_eq:
            total_files_eq += 1

    if project_eq:
        total_projects_eq += 1

print('total_projects:    {}'.format(total_projects))
print('total_projects_eq: {}'.format(total_projects_eq))
print('% affected:        {:.2f}%'.format((total_projects_eq/total_projects)*100))
print('total_files:       {}'.format(total_files))
print('total_files_eq:    {}'.format(total_files_eq))
print('% affected:        {:.2f}%'.format((total_files_eq/total_files)*100))
print('total_reqs:        {}'.format(total_reqs))
print('total_reqs_bare:   {}'.format(total_reqs_bare))
print('% affected:        {:.2f}%'.format((total_reqs_bare/total_reqs)*100))