Version 2.0 of pipista - Unstable branch :)

pipista.py

import os, os.path, sys, urllib2, requests, tempfile, zipfile, shutil, gzip, tarfile

__pypi_base__ = os.path.abspath(os.path.dirname(__file__))

class PyPiError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

def _chunk_report(bytes_so_far, chunk_size, total_size):
    if (total_size != None):
        percent = float(bytes_so_far) / total_size
        percent = round(percent*100, 2)
        print 'Downloaded %d of %d bytes (%0.2f%%)' % (bytes_so_far, total_size, percent)
        if bytes_so_far >= total_size:
            print ''
    else:
        print 'Downloaded %d bytes' % (bytes_so_far)

def _chunk_read(response, chunk_size=32768, report_hook=None, filename=None):
    file_data = []
    if response.info().has_key('Content-Length'):
        total_size = response.info().getheader('Content-Length').strip()
        total_size = int(total_size)
    else:
        # No size
        total_size = None
        if report_hook:
            print '* Warning: No total file size available.'
    if (filename == None) and (response.info().has_key('Content-Disposition')):
        # If the response has Content-Disposition, we take file name from it
        try:
            filename = response.info()['Content-Disposition'].split('filename=')[1]
            if filename[0] == '"' or filename[0] == "'":
                filename = filename[1:-1]
        except Exception:
            sys.exc_clear()
            filename = 'output'
    if (filename == None):
        if report_hook:
            print "* No detected filename, using 'output'"
        filename = 'output'
    bytes_so_far = 0
    while True:
        chunk = response.read(chunk_size)
        bytes_so_far += len(chunk)
        if not chunk:
            break
        else:
            file_data.append(chunk)
        report_hook(bytes_so_far, chunk_size, total_size)
    return (file_data, filename)

def _download(src_dict, print_progress=True):
    headers = {'User-Agent' : 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6;en-US; rv:1.9.2.9) Gecko/20100824 Firefox/3.6.9'}
    if print_progress:
        print '* Downloading:', src_dict['url']
    req = urllib2.Request(src_dict['url'], headers=headers)
    response = urllib2.urlopen(req)
    output = src_dict['url'].split('/')[-1].split('#')[0].split('?')[0]
    if print_progress:
        data,filename = _chunk_read(response, report_hook=_chunk_report, filename=output)
    else:
        data,filename = _chunk_read(response, report_hook=None, filename=output)
    if (len(data) > 0):
        if os.path.exists(filename):
            os.remove(filename)
        try:
            f = open(filename, 'wb')
            for x in data:
                f.write(x)
            f.close()
            if print_progress:
                print '* Saved to:', filename
            return os.path.abspath(filename)
        except Exception:
            if print_progress:
                print '! Error:', sys.exc_info()[1]
            raise
    else:
        if print_progress:
            print '* Error: 0 bytes downloaded, not saved'

def pypi_download(pkg_name, pkg_ver='', print_progress=True):
    import xmlrpclib
    pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
    hits = pypi.package_releases(pkg_name, True)
    if not hits:
        raise PyPiError('No package found with that name')
    if not pkg_ver:
        pkg_ver = hits[0]
    elif not pkg_ver in hits:
        raise PyPiError('That package version is not available')
    hits = pypi.release_urls(pkg_name, pkg_ver)
    if not hits:
        raise PyPiError('No public download links for that version')
    source = ([x for x in hits if x['packagetype'] == 'sdist'][:1] + [None])[0]
    if not source:
        raise PyPiError('No source-only download links for that version')
    return _download(source, print_progress)

def pypi_versions(pkg_name, limit=10, show_hidden=True):
    if not pkg_name:
        return []
    pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
    hits = pypi.package_releases(pkg_name, show_hidden)
    if not hits:
        return []
    if len(hits) > limit:
        hits = hits[:limit]
    return hits

def pypi_search(search_str, limit=5):
    if not search_str:
        return []
    pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
    hits = pypi.search({'name': search_str}, 'and')
    if not hits:
        return []
    hits = sorted(hits, key=lambda pkg: pkg['_pypi_ordering'], reverse=True)
    if len(hits) > limit:
        hits = hits[:limit]
    return hits

def _install_ConfigParser(path='.'):
    # Grab the 2.7.3 version of ConfigParser - even though Pythonista 1.2 is 2.7.0
    r = requests.get('http://hg.python.org/cpython/raw-file/70274d53c1dd/Lib/ConfigParser.py')
    lib_file = os.path.join(path, 'ConfigParser.py')
    with open(lib_file, 'w') as f:
        f.write(r.text)

def _install_xmlrpclib(path='.'):
    # Grab the 2.7.3 version of xmlrpclib - even though Pythonista 1.2 is 2.7.0
    r = requests.get('http://hg.python.org/cpython/raw-file/70274d53c1dd/Lib/xmlrpclib.py')
    lib_file = os.path.join(path, 'xmlrpclib.py')
    with open(lib_file, 'w') as f:
        f.write(r.text)

def _unzip(a_zip=None, path='.'):
    if a_zip is None:
        return
    filename = os.path.abspath(a_zip)
    if not os.path.isfile(filename):
        return
    # PK magic marker check
    f = open(filename)
    try:
        pk_check = f.read(2)
    except Exception:
        pk_check = ''
    finally:
        f.close()
    if pk_check != 'PK':
        print "unzip: %s: does not appear to be a zip file" % a_zip
    else:
        if (os.path.basename(filename).lower().endswith('.zip')):
            altpath = os.path.splitext(os.path.basename(filename))[0]
        else:
            altpath = os.path.basename(filename) + '_unzipped'
        altpath = os.path.join(os.path.dirname(filename), altpath)
        location = os.path.abspath(path)
        if not os.path.exists(location):
            os.makedirs(location)
        zipfp = open(filename, 'rb')
        try:
            zipf = zipfile.ZipFile(zipfp)
            # check for a leading directory common to all files and remove it
            dirnames = [os.path.join(os.path.dirname(x), '') for x in zipf.namelist()]
            common_dir = os.path.commonprefix(dirnames or ['/'])
            # Check to make sure there aren't 2 or more sub directories with the same prefix
            if not common_dir.endswith('/'):
                common_dir = os.path.join(os.path.dirname(common_dir), '')
            for name in zipf.namelist():
                data = zipf.read(name)
                fn = name
                if common_dir:
                    if fn.startswith(common_dir):
                        fn = fn.split(common_dir, 1)[-1]
                    elif fn.startswith('/' + common_dir):
                        fn = fn.split('/' + common_dir, 1)[-1]
                fn = fn.lstrip('/')
                fn = os.path.join(location, fn)
                dirf = os.path.dirname(fn)
                if not os.path.exists(dirf):
                    os.makedirs(dirf)
                if fn.endswith('/'):
                    # A directory
                    if not os.path.exists(fn):
                        os.makedirs(fn)
                else:
                    fp = open(fn, 'wb')
                    try:
                        fp.write(data)
                    finally:
                        fp.close()
        except Exception:
            zipfp.close()
            print "unzip: %s: zip file is corrupt" % a_zip
            return
        zipfp.close()
        return os.path.abspath(location)

def _ungzip(a_gz=None, path='.'):
    fname = 'ungzip'
    if a_gz is None:
        return
    filename = os.path.abspath(a_gz)
    if not os.path.isfile(filename):
        return
    else:
        # '\x1f\x8b\x08' magic marker check
        f = open(filename, 'rb')
        try:
            gz_check = f.read(3)
        except Exception:
            gz_check = ''
        finally:
            f.close()
        if gz_check != '\x1f\x8b\x08':
            print "%s: %s: does not appear to be a gzip file" % (fname,a_gz)
            return
        else:
            if (os.path.basename(filename).lower().endswith('.gz') or os.path.basename(filename).lower().endswith('.gzip')):
                altpath = os.path.splitext(os.path.basename(filename))[0]
            elif os.path.basename(filename).lower().endswith('.tgz'):
                altpath = os.path.splitext(os.path.basename(filename))[0] + '.tar'
            else:
                altpath = os.path.basename(filename) + '_ungzipped'
            location = os.path.join(os.path.abspath(path), altpath)
            if os.path.exists(location):
                # Older file exists somehow, get rid of it
                os.remove(location)
            dirf = os.path.dirname(os.path.dirname(os.path.abspath(location)))
            try:
                if not os.path.exists(dirf):
                    os.makedirs(dirf)
                with open(location, 'wb') as outfile:
                    with gzip.open(filename, 'rb') as gzfile:
                        outfile.write(gzfile.read())
            except Exception:
                print "%s: %s: gzip file is corrupt" % (fname, a_gz)
                return
    return os.path.abspath(location)

def _untar(a_tar=None, path='.'):
    if a_tar is None:
        return
    filename = os.path.abspath(a_tar)
    if not os.path.isfile(filename):
        return
    # 'ustar' magic marker check
    f = open(filename)
    try:
        f.seek(257)
        ustar_check = f.read(5)
    except Exception:
        ustar_check = ''
    finally:
        f.close()
    if ustar_check != 'ustar':
        print "untar: %s: does not appear to be a tar file" % a_tar
        return
    else:
        if (os.path.basename(filename).lower().endswith('.tar')):
            altpath = os.path.splitext(os.path.basename(filename))[0]
        else:
            altpath = os.path.basename(filename) + '_untarred'
        altpath = os.path.join(os.path.dirname(filename), altpath)
        location = os.path.abspath(path)
        if not os.path.exists(location):
            os.makedirs(location)
        try:
            tar = tarfile.open(filename, 'r')
            # check for a leading directory common to all files and remove it
            dirnames = [os.path.join(os.path.dirname(x.name), '') for x in tar.getmembers() if x.name != 'pax_global_header']
            common_dir = os.path.commonprefix(dirnames or ['/'])
            if not common_dir.endswith('/'):
                common_dir = os.path.join(os.path.dirname(common_dir), '')
            for member in tar.getmembers():
                fn = member.name
                if fn == 'pax_global_header':
                    continue
                if common_dir:
                    if fn.startswith(common_dir):
                        fn = fn.split(common_dir, 1)[-1]
                    elif fn.startswith('/' + common_dir):
                        fn = fn.split('/' + common_dir, 1)[-1]
                fn = fn.lstrip('/')
                fn = os.path.join(location, fn)
                dirf = os.path.dirname(fn)
                if member.isdir():
                    # A directory
                    if not os.path.exists(fn):
                        os.makedirs(fn)
                elif member.issym():
                    # skip symlinks
                    continue
                else:
                    try:
                        fp = tar.extractfile(member)
                    except (KeyError, AttributeError):
                        # invalid member, not necessarily a bad tar file
                        continue
                    if not os.path.exists(dirf):
                        os.makedirs(dirf)
                    with open(fn, 'wb') as destfp:
                        shutil.copyfileobj(fp, destfp)
                    fp.close()
        except Exception:
            tar.close()
            print "untar: %s: tar file is corrupt" % a_tar
            return
        tar.close()
        return os.path.abspath(location)

def _install_setuptools(path='.'):
    # Can't use requests - need https access and it's not availabe in Pythonista 1.2
    r = urllib2.urlopen('https://raw.github.com/pudquick/pipista/master/st-lite.zip')
    temp_zip = tempfile.TemporaryFile()
    temp_zip.write(r.read())
    r.close()
    temp_zip.seek(0)
    unzipped = zipfile.ZipFile(temp_zip)
    for name in unzipped.namelist():
        data = unzipped.read(name)
        fn = os.path.abspath(os.path.join(path, name))
        dirn = os.path.dirname(fn)
        if not os.path.exists(dirn):
            os.makedirs(dirn)
        if fn.endswith('/') or fn.endswith('\\'):
            if not os.path.exists(fn):
                os.makedirs(fn)
        else:
            fp = open(fn, 'wb')
            try:
                fp.write(data)
            finally:
                fp.close()
    temp_zip.close()

def _rm(path=None):
        if path is None:
            return
        full_file = os.path.abspath(path).rstrip('/')
        if not os.path.exists(path):
            return
        if (os.path.isdir(full_file)):
            try:
                shutil.rmtree(full_file, True)
                if (os.path.exists(full_file)):
                    return
            except Exception:
                sys.exc_clear()
                return
        else:
            try:
                os.remove(full_file)
            except Exception:
                sys.exc_clear()
                return

def _prep_pipista():
    # This function does some prep work for pipista:
    #  - Sets up the 'pypi-modules' directory if it doesn't exist
    #  - Sets up '.tmp' in pypi-modules if it doesn't exist, for temp storage
    #  - Adds 'pypi-modules' to the import paths
    #  - Makes sure xmlrpclib is installed (fix for Pythonista 1.2)
    #  - Makes sure the minimal setuptools is installed
    # ----------
    # Get pipista location as the relative base for module storage
    lib_dir  = os.path.join(__pypi_base__, 'pypi-modules')
    tmp_dir  = os.path.join(lib_dir, '.tmp')
    if not os.path.exists(lib_dir):
        try:
            os.mkdir(lib_dir)
        except Exception:
            # Fail silently, if we can't make the directory
            sys.exc_clear()
    if not os.path.exists(tmp_dir):
        try:
            os.mkdir(tmp_dir)
        except Exception:
            # Fail silently, if we can't make the directory
            sys.exc_clear()
    # Make sure lib_dir exists before adding it to the paths
    if os.path.exists(lib_dir):
        if lib_dir not in sys.path:
            sys.path += [lib_dir]
    # Attempt to load xmlrpclib - not present in Pythonista 1.2
    try:
        import xmlrpclib
    except ImportError:
        # Doesn't seem to be available - attempt to download it
        sys.exc_clear()
        _install_xmlrpclib(lib_dir)
        import xmlrpclib
    # Attempt to load ConfigParser - not present in Pythonista 1.2
    try:
        import ConfigParser
    except ImportError:
        # Doesn't seem to be available - attempt to download it
        sys.exc_clear()
        _install_ConfigParser(lib_dir)
        import xmlrpclib
    try:
        import distutils.util
        def _fixed_get_platform():
            return sys.platform
        distutils.util.get_platform = _fixed_get_platform
        import setuptools
    except ImportError:
        # Install a lite version of it - just enough for what we need
        sys.exc_clear()
        _install_setuptools(lib_dir)

_prep_pipista()

def _reset_and_enter_tmp(alt_dir=None):
    lib_dir  = os.path.join(__pypi_base__, 'pypi-modules')
    tmp_dir  = os.path.join(lib_dir, '.tmp')
    # Nuke the existing .tmp folder
    _rm(tmp_dir)
    # Prep .tmp folders: archive, unpack
    os.makedirs(tmp_dir)
    os.makedirs(os.path.join(tmp_dir, 'archive'))
    os.makedirs(os.path.join(tmp_dir, 'unpack'))
    if alt_dir is None:
        os.chdir(tmp_dir)
    else:
        # Cleaning up after ourselves, time to return
        os.chdir(alt_dir)

def _py_build(path=None):
    if path is None:
        return
    import distutils.core
    old_stdout = sys.stdout
    old_stderr = sys.stderr
    sys.stdout = tempfile.TemporaryFile()
    sys.stderr = tempfile.TemporaryFile()
    os.chdir(path)
    try:
        result = distutils.core.run_setup('setup.py', ['build_py', '--force'])
    except Exception:
        result = None
    finally:
        sys.stderr = old_stderr
        sys.stdout = old_stdout
    return result

def pypi_install(pkg_name, pkg_ver='', print_progress=True):
    ## EXPERIMENTAL - not guaranteed to work! ##
    # Remeber where we were
    cwd = os.getcwd()
    # Reset build environment and enter it
    _reset_and_enter_tmp()
    # Download the specified package to the archive directory
    os.chdir('archive')
    fn = pypi_download(pkg_name, pkg_ver, print_progress)
    if not fn:
        _reset_and_enter_tmp(cwd)
        return False
    # Successfully downloaded, move back to .tmp
    os.chdir('..')
    res = None
    if fn.lower().endswith('.zip'):
        res = _unzip(os.path.abspath(fn), 'unpack')
    elif (fn.lower().endswith('.tar.gz') or fn.lower().endswith('.tgz')):
        res_tar = _ungzip(os.path.abspath(fn), 'archive')
        if res_tar:
            # Got the tar file, now to unpack it
            res = _untar(os.path.abspath(res_tar), 'unpack')
    elif fn.lower().endswith('.tar'):
        res = _untar(os.path.abspath(fn), 'unpack')
    if res:
        # Assuming we got to this point, we now have a directory, with possible sub-dirs, with a setup.py
        # Have to find that setup.py, search top down for first one
        # Pick a sane default
        setup_dir = res
        for (path, dirs, files) in os.walk(res):
            if 'setup.py' in set([x.lower() for x in files]):
                setup_dir = path
                break
        setup_dir = os.path.abspath(setup_dir)
        print "* setup.py found here:", setup_dir
        # Attempt a pure python build
        print "* Compiling pure python modules ..."
        result = _py_build(setup_dir)
        if result:
            # Should be contents inside setup_dir/build/lib - merge them into pypi-modules
            build_dir = os.path.join(setup_dir, 'build/lib')
            if os.path.exists(build_dir):
                # Get the files and directories in it
                os.chdir(build_dir)
                (path, dirs, files) = os.walk(build_dir).next()
                lib_dir = os.path.abspath(os.path.join(__pypi_base__, 'pypi-modules'))
                for a_file in files:
                    print "* Installing module: %s ..." % a_file
                    target = os.path.join(lib_dir, a_file)
                    if os.path.exists(target):
                        _rm(target)
                    shutil.copyfile(a_file, target)
                for a_dir in dirs:
                    print "* Installing module: %s ..." % a_dir
                    target = os.path.join(lib_dir, a_dir)
                    if os.path.exists(target):
                        _rm(target)
                    shutil.copytree(a_dir, target)
        _reset_and_enter_tmp(cwd)
        return True
    else:
        print "* ERROR: Something went wrong"
    _reset_and_enter_tmp(cwd)
    return False

How does this work

Would be nice to support grabbing packages from URL's http/git

For what it's worth, I think I figured out the exec f.read() in g,l issue (where imports don't work). This should be exec f.read() in g, g. I don't claim to fully understand why, but your example the forum of just doing exec on a string with an import also fails if wrapped inside a function.
Not sure of an easy way to fix that, guess we need to patch setuptools itself, which maybe is easy given you already have a lite version.

I want to install module "wifi", but unfortunatelly, I cannot call to run file "setup.py" via "distutils.core.run_setup".
The error message is " Skipping copying files to /etc/... , no write access.
How can I resolve this issue?

Traceback (most recent call last):
File "/private/var/mobile/Containers/Shared/AppGroup/9EF748E7-E9EC-4528-ADE7-7E6D6FC841DD/Pythonista3/Documents/pipista_install.py", line 2, in
pipista.pypi_install('pandas')
File "/private/var/mobile/Containers/Shared/AppGroup/9EF748E7-E9EC-4528-ADE7-7E6D6FC841DD/Pythonista3/Documents/pipista.py", line 460, in pypi_install
fn = pypi_download(pkg_name, pkg_ver, print_progress)
File "/private/var/mobile/Containers/Shared/AppGroup/9EF748E7-E9EC-4528-ADE7-7E6D6FC841DD/Pythonista3/Documents/pipista.py", line 95, in pypi_download
hits = pypi.release_urls(pkg_name, pkg_ver)
File "/var/containers/Bundle/Application/CBE5150F-2F53-4F25-8FE0-DC5236FFA836/Pythonista3.app/Frameworks/Py2Kit.framework/pylib/xmlrpclib.py", line 1244, in call
return self.__send(self.__name, args)
File "/var/containers/Bundle/Application/CBE5150F-2F53-4F25-8FE0-DC5236FFA836/Pythonista3.app/Frameworks/Py2Kit.framework/pylib/xmlrpclib.py", line 1603, in __request
verbose=self.__verbose
File "/var/containers/Bundle/Application/CBE5150F-2F53-4F25-8FE0-DC5236FFA836/Pythonista3.app/Frameworks/Py2Kit.framework/pylib/xmlrpclib.py", line 1284, in request
return self.single_request(host, handler, request_body, verbose)
File "/var/containers/Bundle/Application/CBE5150F-2F53-4F25-8FE0-DC5236FFA836/Pythonista3.app/Frameworks/Py2Kit.framework/pylib/xmlrpclib.py", line 1317, in single_request
return self.parse_response(response)
File "/var/containers/Bundle/Application/CBE5150F-2F53-4F25-8FE0-DC5236FFA836/Pythonista3.app/Frameworks/Py2Kit.framework/pylib/xmlrpclib.py", line 1494, in parse_response
return u.close()
File "/var/containers/Bundle/Application/CBE5150F-2F53-4F25-8FE0-DC5236FFA836/Pythonista3.app/Frameworks/Py2Kit.framework/pylib/xmlrpclib.py", line 801, in close
raise Fault(**self._stack[0])
Fault: <Fault -32500: 'HTTPTooManyRequests: The action could not be performed because there were too many requests by the client. Limit may reset in 1 seconds.'>