remove_dup_pip_requirements.py

#!/usr/bin/env python

# Quick and dirty script to scrub pip requirements files of duplicate requirements

import sys
import fileinput

from requirements.requirement import Requirement

seen = set()
out = []
for line in fileinput.input():
    try:
        name = Requirement.parse(line).name
        if name in seen:
            continue
        seen.add(name)
        sys.stdout.write(line)
    except ValueError as e:
        sys.stdout.write(line)