Skip to content

Instantly share code, notes, and snippets.

@matham

matham/kivy-env.py

Last active August 29, 2015 14:05
Show Gist options
  • Save matham/f6f7aa83d803dbde25c4 to your computer and use it in GitHub Desktop.
Save matham/f6f7aa83d803dbde25c4 to your computer and use it in GitHub Desktop.
Download ZIP
Create windows portable python for kivy
Raw
kivy-env.py
from __future__ import print_function
import subprocess
import sys
import os
from os import makedirs, listdir, remove, rename
from os.path import exists, join, abspath, isdir, isfile, splitext
import argparse
from subprocess import Popen, PIPE
from shutil import rmtree, copytree, copy2
from glob import glob
import hashlib
import re
from zipfile import ZipFile
try:
from urllib.request import urlretrieve
except ImportError:
from urllib import urlretrieve
def report_hook(block_count, block_size, total_size):
p = block_count * block_size * (100.0 / total_size if total_size else 1)
print("\b\b\b\b\b\b\b\b\b", "%06.2f%%" % p, end=' ')
def exec_binary(status, cmd, env=None, cwd=None, shell=True):
print(status)
print(' '.join(cmd))
proc = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd, shell=shell)
a, b = proc.communicate()
print(a, end='')
print(b, end='')
class WindowsPortablePythonBuild(object):
'''Custom build command that builds portable win32 python
and kivy deps.
'''
pydb = {
'py2.7.8_x86': ('https://www.python.org/ftp/python/2.7.8/python-2.7.8.msi',
'ef95d83ace85d1577b915dbd481977d4'),
'py2.7.8_x64': ('https://www.python.org/ftp/python/2.7.8/python-2.7.8.amd64.msi',
'38cadfcac6dd56ecf772f2f3f14ee846'),
'py3.4.1_x86': ('https://www.python.org/ftp/python/3.4.1/python-3.4.1.msi',
'4940c3fad01ffa2ca7f9cc43a005b89a'),
'py3.4.1_x64': ('https://www.python.org/ftp/python/3.4.1/python-3.4.1.amd64.msi',
'25440653f27ee1597fd6b3e15eee155f'),
'py2.7.6_x86': ('https://www.python.org/ftp/python/2.7.6/python-2.7.6.msi',
'ac54e14f7ba180253b9bae6635d822ea')}
pip_deps = ['cython', 'docutils', 'pygments', 'requests', 'plyer',
'kivy-garden', 'wheel']
pywin_baseurl = ''
pygame_baseurl = ''
dist_dir = None
temp_dir = ''
build_pythons = []
mingw = ''
mingw64 = ''
noclean = False
gendef = ''
def parse_args(self):
parser = argparse.ArgumentParser(description='Generates portable package'
' for usage with kivy. It includes python and MinGW for 64 and 32 bit '
'platforms. To use, you specify the python versions to build, paths '
'to MinGW, a path to local copies of pygame binaries, and '
'the path to gendef if building 64 bit.')
parser.add_argument("--dir", help='path of dist directory to use '
'for building the portable python, the end result will be output '
'to this directory. Defaults to cwd.', default=os.getcwd())
parser.add_argument("pythons", help='The '
'pythons to generate, in the format of `pyversion_arch`, where version'
' is e.g. 2.7.6 and arch is either x86 or x64. For example: '
'py2.7.8_x86.')
parser.add_argument("mingw", help='Path to MinGW. '
'To get it, download mingw-get-setup.exe from '
'http://sourceforge.net/projects/mingw/files/Installer/mingw-get-setup.exe/download.'
'\nTo install, create a directory and run the installer which will install'
'a command line or gui mingw package manager. Then you install the required'
'packages (gcc-core) using that manager. Pass the name of this '
'directory to this option.')
parser.add_argument("mingw64", help='Path to 64 bit MinGW. '
'To get it, download the latest 64 bit mingw from '
'http://sourceforge.net/projects/mingw-w64/files/Toolchains%%20targetting%%20Win64/Personal%%20Builds/mingw-builds/.'
'\nTo install, just unzip the downloaded file to a directory and pass the name of'
'that directory to this option.')
parser.add_argument("--pywin", help='Initial path of the pywin '
'binaries url on sourceforge. Defaults to '
'http://iweb.dl.sourceforge.net/project/pywin32/pywin32/Build%20219/',
default='http://iweb.dl.sourceforge.net/project/pywin32/pywin32/Build%20219/')
parser.add_argument("pygame", help='Path to pygame binaries downloaded '
'from http://www.lfd.uci.edu/~gohlke/pythonlibs/#pygame')
parser.add_argument("--gendef", help='Path to the gendef executable '
'you can get it from http://sourceforge.net/projects/mingw/files/MinGW/Extension/gendef/gendef-1.0.1346/'
'This path should have a file called gendef.exe. If not specified'
" it's assumed to be in MinGW/bin.", default='')
parser.add_argument("--noclean", help='Whether to remove the python '
'and MinGW build if it already exists', action="store_true")
args = parser.parse_args()
self.dist_dir = base = abspath(args.dir)
self.temp_dir = join(base, 'temp')
pydb = self.pydb
build_py = []
for py in args.pythons.split(','):
py = py.strip()
if py not in pydb:
raise Exception('Python {} not recognized'.format(py))
build_py.append((py, pydb[py]))
self.build_pythons = build_py
self.mingw = abspath(args.mingw)
self.mingw64 = abspath(args.mingw64)
self.pywin_baseurl = args.pywin
self.pygame_path = args.pygame
self.noclean = args.noclean
gendef = args.gendef
if gendef:
self.gendef = join(abspath(gendef), 'gendef.exe')
else:
self.gendef = 'gendef.exe'
def run(self):
self.parse_args()
width = self.width = 30
print("-" * width)
print("Generating python distribution")
print("-" * width)
print("\nPreparing Build...")
print("-" * width)
dist_dir = self.dist_dir
print('Working in {}'.format(dist_dir))
temp = self.temp_dir
if exists(temp):
print("*Removing old temp dir, {}".format(temp))
rmtree(temp, ignore_errors=True)
print('Creating temp directory {}'.format(temp))
makedirs(temp)
py_pat = re.compile('py([0-9])\.([0-9])\.([0-9])_x([0-9]+)')
for pyname, (url, md5) in self.build_pythons:
print('\n')
print("-" * width)
print("Preparing python {}".format(pyname))
print("-" * width)
name = pyname.replace('.', '')
build_path = join(dist_dir, name)
m = re.match(py_pat, pyname)
pyver = m.groups()[:3]
arch = int(m.group(4))
patch_py = False
pydir = join(build_path, 'Python')
if not self.noclean and exists(build_path):
print("*Cleaning old build dir, {}".format(build_path))
rmtree(build_path, ignore_errors=True)
if not exists(build_path):
print("*Creating build directory: {}".format(build_path))
makedirs(build_path)
if not exists(pydir):
self.get_python(width, build_path, url, md5)
patch_py = arch == 64
mingw = join(build_path, 'MinGW')
if not exists(mingw):
src = self.mingw if arch == 86 else self.mingw64
print('Copying Mingw from {} to {}'.format(src, mingw))
copytree(src, mingw)
env = os.environ.copy()
env['PATH'] = '{};{};{};{}'.format(pydir, join(mingw, 'bin'),
join(pydir, 'Scripts'), env['PATH'])
env['PYTHONPATH'] = ''
if patch_py:
print('Patching python')
self.patch_python_x64(pydir, env, pyver)
if arch == 86:
pywin = 'pywin32-219.win32-py{}.{}.exe'.format(*pyver[:2])
pygame = 'pygame-1.9.2a0.win32-py{}.{}.exe'.format(*pyver[:2])
glew = 'glew-1.5.7-win32.zip'
else:
pywin = 'pywin32-219.win-amd64-py{}.{}.exe'.format(*pyver[:2])
pygame = 'pygame-1.9.2a0.win-amd64-py{}.{}.exe'.format(*pyver[:2])
glew = 'glew-1.5.7-win64.zip'
self.get_glew(pydir, mingw, glew, arch, env)
self.get_pip_deps(pydir, env, self.pywin_baseurl + pywin,
join(self.pygame_path, pygame))
print('Removing temp directory')
rmtree(temp, ignore_errors=True)
print('Done')
def get_python(self, width, build_path, url, md5):
print("*Downloading: {}".format(url))
print("Progress: 000.00%", end=' ')
f, _ = urlretrieve(url, join(self.temp_dir, url.split('/')[-1]),
reporthook=report_hook)
print(" [Done]")
print("Verifying MD5", end=' ')
with open(f, 'rb') as fd:
md5_read = hashlib.md5(fd.read()).hexdigest()
if md5_read != md5:
raise Exception('MD5 not verified. Gotten: {}, expected: {}'.
format(md5_read, md5))
print(" Done")
pydir = join(build_path, 'Python')
makedirs(pydir)
exec_binary("*Extracting python to {}".format(pydir),
['msiexec', '/a', f, '/qb', 'TARGETDIR={}'.format(pydir)],
shell=False)
print("\nCleaning python")
print("-" * width)
for f in listdir(pydir):
if f.endswith('.msi'):
f = join(pydir, f)
print('Removing {}'.format(f))
remove(f)
break
for d in ('tcl', 'Doc', join('Lib', 'test'), join('Lib', 'lib-tk'),
join('Lib', 'idlelib')):
print('Removing {}'.format(d))
rmtree(join(pydir, d), ignore_errors=True)
ignore_list = ()
for f in listdir(join(pydir, 'Lib')):
if f in ignore_list:
continue
f = join(pydir, 'Lib', f)
if not isdir(f):
continue
dirs = os.listdir(f)
if 'test' in dirs and isdir(join(f, 'test')):
f = join(f, 'test')
elif 'tests' in dirs and isdir(join(f, 'tests')):
f = join(f, 'tests')
else:
continue
print('Removing {}'.format(f))
rmtree(f, ignore_errors=True)
for f in listdir(join(pydir, 'DLLs')):
if (f.startswith('tcl') or f.startswith('tk') or
f.startswith('_tkinter')):
f = join(pydir, 'DLLs', f)
print('Removing {}'.format(f))
remove(f)
f = join(pydir, 'Lib', 'distutils', 'distutils.cfg')
print('Writing {}'.format(f))
with open(f, 'w') as fd:
fd.write('[build]\ncompiler=mingw32\n')
makedirs(join(pydir, 'Scripts'))
def patch_python_x64(self, pydir, env, pyver):
gendef = self.gendef
libs = join(pydir, 'libs')
py = 'python{}{}'.format(*pyver[:2])
pylib = py + '.lib'
pydll = join(pydir, py + '.dll')
pydef = join(libs, py + '.def')
# see http://bugs.python.org/issue4709 and
# http://ascend4.org/Setting_up_a_MinGW-w64_build_environment
rename(join(libs, pylib), join(libs, 'old_' + pylib))
exec_binary('Gendefing ' + pydll, [gendef, pydll], env, libs)
exec_binary('Generating libpython.a',
['dlltool', '--dllname', pydll, '--def', pydef,
'--output-lib', 'lib' + py + '.a'], env, libs, shell=True)
remove(pydef)
print('Getting python pyconfig.h patch')
url = 'http://bugs.python.org/file12411/mingw-w64.patch'
include = join(pydir, 'include')
patch_name = url.split('/')[-1]
patch, _ = urlretrieve(url, join(include, patch_name))
exec_binary('Patching {}\\pyconfig.h'.format(include),
['git', 'apply', patch_name], env, include, shell=True)
remove(patch)
# see http://bugs.python.org/issue16472
print('Patching cygwinccompiler.py')
cyg = join(pydir, 'Lib', 'distutils', 'cygwinccompiler.py')
with open(cyg) as fd:
lines = fd.readlines()
with open(cyg, 'w') as fd:
for line in lines:
if line == ' self.dll_libraries = get_msvcr()\n':
fd.write(' #self.dll_libraries = get_msvcr()\n')
else:
fd.write(line)
def get_pip_deps(self, pydir, env, pywin, pygame):
width = self.width
temp_dir = self.temp_dir
py = join(pydir, 'python.exe')
print('Getting pip and easy install')
url = 'https://bootstrap.pypa.io/get-pip.py'
pip, _ = urlretrieve(url, join(temp_dir, url.split('/')[-1]))
url = 'https://bootstrap.pypa.io/ez_setup.py'
ez, _ = urlretrieve(url, join(temp_dir, url.split('/')[-1]))
exec_binary('Installing pip', [py, pip], env, pydir, shell=True)
exec_binary('Installing easy install', [py, ez], env, pydir, shell=True)
pip = join(pydir, 'Scripts', 'pip.exe')
for mod in self.pip_deps:
exec_binary('Installing {}'.format(mod), [pip, 'install', mod],
env, pydir, shell=True)
wheel = join(pydir, 'Scripts', 'wheel.exe')
print("Downloading pywin32. Progress: 000.00%", end=' ')
pywin, _ = urlretrieve(pywin, join(temp_dir, pywin.split('/')[-1]),
reporthook=report_hook)
print(' [Done]')
a = listdir(temp_dir)
exec_binary('Converting {} to wheel'.format(pywin),
[wheel, 'convert', pywin], env, temp_dir, shell=True)
b = listdir(temp_dir)
exec_binary('Converting {} to wheel'.format(pygame),
[wheel, 'convert', pygame], env, temp_dir, shell=True)
# get the path of the wheel file generated
pywin = join(temp_dir, list(set(b) - set(a))[0])
pygame = join(temp_dir, list(set(listdir(temp_dir)) - set(b))[0])
exec_binary('Installing the wheel {}'.format(pywin),
[wheel, 'install', '--force', pywin], env, pydir, shell=True)
exec_binary('Installing the wheel {}'.format(pygame),
[wheel, 'install', pygame], env, pydir, shell=True)
def get_glew(self, pydir, mingw, glew, arch, env):
temp_dir = self.temp_dir
url = 'http://iweb.dl.sourceforge.net/project/glew/glew/1.5.7/' + glew
print("*Getting glew. Downloading: {}".format(url))
print("Progress: 000.00%", end=' ')
f, _ = urlretrieve(url, join(temp_dir, glew), reporthook=report_hook)
print(" [Done]")
z = join(temp_dir, splitext(glew)[0])
print('Extracting glew {}'.format(f))
with open(f, 'rb') as fd:
ZipFile(fd).extractall(z)
z = join(z, 'glew-1.5.7')
print('Distributing glew to mingw and python')
include = join(mingw, 'include', 'GL')
py_include = join(pydir, 'include', 'GL')
if not exists(include):
makedirs(include)
makedirs(py_include)
for f in glob(join(z, 'include', 'GL', '*')):
try:
copy2(f, include)
except:
pass
copy2(f, py_include)
lib = join(mingw, 'lib')
copy2(join(z, 'bin', 'glew32.dll'), pydir)
try:
copy2(join(z, 'bin', 'glew32.dll'), lib)
copy2(join(z, 'lib', 'glew32.lib'), lib)
copy2(join(z, 'lib', 'glew32s.lib'), lib)
except:
pass
gendef = self.gendef
libs = join(pydir, 'libs')
glew_dll = join(z, 'bin', 'glew32.dll')
glew_def = join(libs, 'glew32.def')
exec_binary('Gendefing ' + glew_dll, [gendef, glew_dll], env, libs)
exec_binary('Generating libglew32.a',
['dlltool', '--dllname', glew_dll, '--def', glew_def,
'--output-lib', 'libglew32.a'], env, libs)
remove(glew_def)
if __name__ == '__main__':
WindowsPortablePythonBuild().run()
Raw
notes.txt
Some notes:
- It does not include gstreamer
- It copies the glew.dll to Python/ because in the kivy.bat, Python/dlls or Python/libs are not added to the path, only Python/is.
- You need to manually copy msvcr100.dll (from system32, or WoWsystem32 for x86, x64, respectively) for py3, and msvcr90.dll for py2 into the Pyhton/? directory.
- While installing a file is named setuptools-xx.zip is created in Python/, remove it.
To use it
- you need to download pygame (http://www.lfd.uci.edu/~gohlke/pythonlibs/#pygame) 64/32, py3/py2 into a directory
- download mingw and mingw64 (see links in the script above) and extract/install them somewhere. These installs are quite large, so trim as needed
- download gendef from link in the script.
- call the script with the target dir, the versions of python to install, paths to mingw, mingw64, path to gendef (if not on the path already), and the path to the pygame bins.
An example is: python kivy-env.py --dir C:\kivy-env py2.7.8_x64,py2.7.8_x86,py3.4.1_x86,py3.4.1_x64 C:\kivy-env\MinGW C:\kivy-env\MinGW64 C:\kivy-env\pygame --gendef C:\kivy-env
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment