pypi_search_parser.py

from HTMLParser import HTMLParser
import sys
import pprint
import urllib
from pip.commands.search import highest_version
import pkg_resources

class PyPISearchParser(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.in_results = False
        self.result = None
        self.results = []
        self.data_target = None
        self.saw_table_heading = False

    def handle_starttag(self, tag, attrs):
        if tag == 'table':
            self.in_results = True
        elif tag == 'tr':
            self.result = {}
        elif tag == 'td':
            if dict(attrs).get('id') == 'last':
                self.in_results = False
            else:
                data_target = ['name', 'version', 'score', 'summary'][len(self.result)]
                self.result[data_target] = ''
                self.data_target = data_target

    def handle_endtag(self, tag):
        if tag == 'table':
            self.in_results = False
        elif tag == 'tr':
            if self.in_results:
                # skip first row (heading)
                if len(self.results) > 0:
                    self.results.append(self.result)
                else:
                    if self.saw_table_heading:
                        self.results.append(self.result)
                    else:
                        self.saw_table_heading = True

                self.result = None
        elif tag == 'td':
            self.data_target = None

    def handle_data(self, data):
        if self.data_target == 'name':
            self.result['name'] += data
            self.data_target = 'version'
            self.result['version'] = ''

        elif self.data_target:
            self.result[self.data_target] += data

def main():
    query = ' '.join(sys.argv[1:])
    query_args = urllib.urlencode({'action': 'search', 'term': query, 'submit': 'search' })
    query_url = "http://pypi.python.org/pypi?:%s" % query_args

    handle = urllib.urlopen(query_url)
    data =  handle.read()

    parser = PyPISearchParser()
    parser.feed(data)

    # versions must be in list form to pass to print_results
    results = []
    for result in parser.results:
        result['versions'] = [result['version']]
        del result['version']
        results.append(result)

    print_results(results)


# copied from pip source. will use the pip one in final version.
# quick hack - print to stdout instead of logger
def print_results(hits, name_column_width=25, terminal_width=None):
    installed_packages = [p.project_name for p in pkg_resources.working_set]
    for hit in hits:
        name = hit['name']
        summary = hit['summary'] or ''
        if terminal_width is not None:
            # wrap and indent summary to fit terminal
            summary = textwrap.wrap(summary, terminal_width - name_column_width - 5)
            summary = ('\n' + ' ' * (name_column_width + 3)).join(summary)
        line = '%s - %s' % (name.ljust(name_column_width), summary)
        try:
            print(line)
            if name in installed_packages:
                dist = pkg_resources.get_distribution(name)
                latest = highest_version(hit['versions'])
                if dist.version == latest:
                    print('  INSTALLED: %s (latest)' % dist.version)
                else:
                    print('  INSTALLED: %s' % dist.version)
                    print('  LATEST:    %s' % latest)
        except UnicodeEncodeError:
            pass

if __name__ == '__main__':
    main()