mikewest/build.py

## build.py
#!/usr/bin/env python
# encoding: utf-8;

from __future__ import with_statement

import os
import errno
import sys
import re
import subprocess
import hashlib
import shutil
from optparse import OptionParser

try:
    from yaml import load, CLoader as Loader
except ImportError:
    from yaml import Loader, Dumper

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError, exc:
        if exc.errno == errno.EEXIST:
            pass
        else: raise

def versionhash( buffer ):
    return hashlib.sha1( buffer ).hexdigest()[0:8]

def hashfile( file ):
    with open( file, 'rb' ) as f:
        return versionhash( f.read() )

class StaticAssetBuilder(object):
    def __init__( self, settings ):
        self.urls_seen = {}
        self.settings  = settings

    def log( self, message ):
        if self.settings[ 'VERBOSE' ]:
            print message

    def process_urls( self, line, source ):
        """
            Given a line of text, find all strings of the form
            `/static_assets/dev/...`, and generate appropriate
            live URLs.

            For our purposes, that means:

            1.  Assign the file a CDN server (reusing previously assigned
                servers if we've already seen the file).

            2.  Strip the dev root pattern (`/static_assets/dev`), and replace
                with the live pattern (`/static_assets/build`)

            3.  Tag each file with it's content hash.
        """
        urls_seen = self.urls_seen
        settings  = self.settings

        def replacer( matchobj ):
            url = matchobj.group(0)
            if not url in urls_seen:
                static_file = url.replace( matchobj.group( 1 ), settings[ 'SOURCE_ROOT' ] )
                if self.settings[ 'CDN' ]:
                    if not os.path.isfile( static_file ):
                        print "WARNING: You've included `%s` in `%s`.  It doesn't seem to exist." % ( static_file, source )
                        hash = '00000000'
                    else:
                        hash = hashfile( static_file )
                    cdn = settings[ 'CDN_SERVERS' ][ len( urls_seen ) % len( settings[ 'CDN_SERVERS' ] ) ]
                    hashed_file = re.sub( r'(.\w+)$', r'.%s\1' % hash, matchobj.group( 2 ) )
                    urls_seen[ url ] = "http://%s%s/%s" % ( cdn, settings[ 'LIVE_ROOT_PATTERN' ], hashed_file )
                    static_file_dest = os.path.join( settings[ 'OUTPUT_ROOT' ], hashed_file )
                else:
                    urls_seen[ url ] = url
                    static_file_dest = os.path.join( settings[ 'OUTPUT_ROOT' ], matchobj.group( 2 ) )

                mkdir_p( os.path.dirname( static_file_dest ) )
                shutil.copy( static_file, static_file_dest )

            return urls_seen[ url ]

        return re.sub(
            self.settings[ 'DEV_ROOT_PATTERN' ],
            replacer,
            line)

    def static_combine( self, end_file, to_combine, delimiter="\n/* Begin: %s */\n" ):
        self.log( "*   Building `%s`" % os.path.basename( end_file ) )

        if end_file.find( '.css' ) != -1:
            mode = 'CSS'
        else:
            mode = 'JS'

        processed_output = ''
        if mode == 'CSS':
            processed_output = """@charset "UTF-8";\n"""

        for static_file in to_combine:
            if os.path.isfile( static_file ):
                if delimiter:
                    processed_output += delimiter % os.path.split( static_file )[ 1 ]
                with open( static_file, 'r' ) as f:
                    for line in f:
                        processed_output += self.process_urls( line, static_file )

        if processed_output:
            with open( end_file, 'w' ) as combo:
                combo.write( processed_output )

            if self.settings[ 'COMPRESS' ]:
                try:
                    command =   self.settings[ 'COMPRESSION_COMMAND' ] % end_file
                    proc    =   subprocess.Popen( command, shell=True, stdout=subprocess.PIPE )
                    output  =   proc.communicate()[ 0 ]
                    hash    =   versionhash( output )
                    with open( re.sub( r'(\.\w+)$', r'.%s\1' % hash, end_file ), 'w' ) as minified:
                        minified.write( output )
                except AttributeError, error:
                    raise CommandError("COMPRESSION_COMMAND not set")
                except TypeError, error:
                    raise CommandError("No string substitution provided for the input file to be passed to the argument ('cmd %s')")

    def build( self ):
        for filetype in [ 'js', 'css' ]:
            if filetype in self.settings[ 'FILES' ]:
                to_build = self.settings[ 'FILES' ][ filetype ]
                build_root = os.path.join( self.settings[ 'OUTPUT_ROOT' ], filetype )
                mkdir_p( build_root )
                for filename, filelist in to_build.items():
                    self.static_combine(
                        end_file=os.path.join( build_root, filename ),
                        to_combine=[ os.path.join( self.settings[ 'SOURCE_ROOT' ], filetype, f) for f in filelist ]
                    )

def main(argv=None):
    if argv is None:
        argv = sys.argv

    default_root = os.path.dirname(os.path.abspath(__file__))

    parser = OptionParser(usage="Usage: %prog [options]", version="%prog 0.1")
    parser.add_option(  "--verbose",
                        action="store_true", dest="verbose_mode", default=False,
                        help="Verbose mode")

    parser.add_option(  "-o", "--output",
                        action="store",
                        dest="output_root",
                        default=os.path.join(default_root, '../build'),
                        help="Directory to which output will be written")

    parser.add_option(  "-s", "--source",
                        action="store",
                        dest="source_root",
                        default=os.path.join(default_root, "../src"),
                        help="Directory from which source files will be read")

    parser.add_option(  "-c", "--compress",
                        action="store_true",
                        dest="compress",
                        default=False,
                        help="Compress the generated files using the specified compression command")

    parser.add_option(  "--cdn",
                        action="store_true",
                        dest="use_cdn",
                        default=False,
                        help="Generate CDN URLs for static assets in generated files.")
    (options, args) = parser.parse_args()

    with open( "%s/_config/build.yaml" % options.source_root, 'r' ) as f:
        settings = load( f )

    settings[ 'COMPRESSION_COMMAND' ] = settings[ 'COMPRESSION_COMMAND' ] % default_root

    settings[ 'VERBOSE' ]       = options.verbose_mode
    settings[ 'CDN' ]           = options.use_cdn
    settings[ 'COMPRESS']       = options.compress
    settings[ 'OUTPUT_ROOT' ]   = options.output_root
    settings[ 'SOURCE_ROOT' ]   = options.source_root

    builder = StaticAssetBuilder( settings )
    builder.build()


if __name__ == "__main__":
    sys.exit(main())

## config.py
COMPRESSION_COMMAND:   'java -jar "%s/lib/yuicompressor-2.4.2.jar" "%%s"'
CDN_SERVERS:
    -   "pix1.sueddeutsche.de"
    -   "pix2.sueddeutsche.de"
DEV_ROOT_PATTERN:       '(/static_assets/dev)/([\-_a-zA-Z0-9\.\/]+)'
LIVE_ROOT_PATTERN:      '/static_assets/build'
FILES:
    js:
        "grid.js":
        -   "copyright.js"
        -   "third-party/jquery-1.4.2.js"
        -   "third-party/jquery.onAvailable-1.0.js"
        -   "third-party/linktracker.js"
        -   "third-party/swfobject.js"
        -   "sde/sde.js"
        -   "advertisement/advertisement.js"
        -   "advertisement/google.js"
        -   "advertisement/sitestat.js"
        -   "advertisement/yahoo.js"
#
#   JS Utils
#
        -   "utils/utils.js"
        -   "utils/config.js"
        -   "utils/embedflash.js"
        -   "utils/interval.js"
        -   "utils/listtoselect.js"
        -   "utils/popup.js"
#
#   Widget Base Classes
#
        -   "widget/widget.js"
        -   "widget/dropdowncontrol.js"
        -   "widget/lightboxcontrol.js"
        -   "widget/overlay.js"
        -   "widget/tabcontrol.js"

#
#   Widget Implementations
#
        -   "widget/bookmarking.js"
        -   "widget/dynamiclist.js"
        -   "widget/flashelement.js"
        -   "widget/login.js"
        -   "widget/mailtofriend.js"
        -   "widget/poll.js"
        -   "widget/projector.js"
        -   "widget/tabs.js"
        -   "widget/video.js"
        -   "widget/weitereangebote.js"
        -   "widget/zoomable.js"
#
#   Initalization
#
        -   "init/siteheader.js"
        -   "init/article3.js"
        -   "init/homepage.js"
    css:
        "grid.css":
        -   "copyright.css"
        -   "yui/reset.css"
        -   "yui/fonts.css"
        -   "decoration.css"
        -   "grid.css"
        -   "sidebar.css"
        -   "site/siteheader.css"
        -   "site/sitefooter.css"
        -   "colors/colors-ressort.css"
        -   "colors/colors-special.css"
        -   "article/article.css"
        -   "article/header.css"
        -   "article/footer.css"
        -   "gallery/gallery.css"
        -   "gallery/header.css"
        -   "gallery/footer.css"
        -   "modules/paging.css"
        -   "modules/modules.css"
        -   "modules/basebox.css"
        -   "modules/basebox_columns.css"
        -   "modules/basebox_columns_infothek.css"
        -   "modules/basebox_columns_sparmeister.css"
        -   "modules/basebox_columns_themelist.css"
        -   "modules/basebox_comments.css"
        -   "modules/basebox_impression.css"
        -   "modules/basebox_singlelink.css"
        -   "modules/basebox_inlinevideo.css"
        -   "modules/basebox_dynamiclist.css"
        -   "modules/basebox_focusedteaser.css"
        -   "modules/basebox_gallerylist.css"
        -   "modules/basebox_imageblock.css"
        -   "modules/basebox_imageblock_related.css"
        -   "modules/basebox_infothek.css"
        -   "modules/basebox_liveticker.css"
        -   "modules/basebox_opinion.css"
        -   "modules/basebox_szpromo.css"
        -   "modules/basebox_tagcloud.css"
        -   "modules/basebox_mailtofriend.css"
        -   "modules/mehrzumthema.css"
        -   "modules/themenbox.css"
        -   "modules/inlinemap.css"
        -   "modules/bildbanderolle.css"
        -   "modules/breakingnewsletter.css"
        -   "modules/headslot.css"
        -   "modules/headslot_image.css"
        -   "modules/bildergaleriebox-stoerer.css"
        -   "modules/projector.css"
        -   "modules/tabbox.css"
        -   "modules/socialbookmarking.css"
        -   "modules/servicebox.css"
        -   "modules/xlinks.css"
        -   "modules/toplisten.css"
        -   "modules/poll.css"
        -   "modules/tabbox_finance.css"
        -   "modules/teaserlist.css"
        -   "modules/teaserlist_breakingnews.css"
        -   "modules/teaserlist_metadata.css"
        -   "modules/teaserlist_paging.css"
        -   "modules/teaserlist_topteaser.css"
        -   "modules/themehighlights.css"
#   Ressortblock, _then_ stockchart, _then_ basebox_stockchart.
        -   "modules/ressortblock.css"
        -   "modules/stockchart.css"
        -   "modules/basebox_stockchart.css"
        -   "modules/hp_sonderthemen.css"
#   Whisper, _then_ basebox_whisper.
        -   "modules/whisper.css"
        -   "modules/basebox_whisper.css"
#
#   Advertisement Styles.  These are my favourites.
#
        -   "advertisement/yahoo.css"
        -   "advertisement/google.css"
        -   "advertisement/ivw.css"
	#!/usr/bin/env python
	# encoding: utf-8;

	from __future__ import with_statement

	import os
	import errno
	import sys
	import re
	import subprocess
	import hashlib
	import shutil
	from optparse import OptionParser

	try:
	from yaml import load, CLoader as Loader
	except ImportError:
	from yaml import Loader, Dumper

	def mkdir_p(path):
	try:
	os.makedirs(path)
	except OSError, exc:
	if exc.errno == errno.EEXIST:
	pass
	else: raise

	def versionhash( buffer ):
	return hashlib.sha1( buffer ).hexdigest()[0:8]

	def hashfile( file ):
	with open( file, 'rb' ) as f:
	return versionhash( f.read() )

	class StaticAssetBuilder(object):
	def __init__( self, settings ):
	self.urls_seen = {}
	self.settings = settings

	def log( self, message ):
	if self.settings[ 'VERBOSE' ]:
	print message

	def process_urls( self, line, source ):
	"""
	Given a line of text, find all strings of the form
	`/static_assets/dev/...`, and generate appropriate
	live URLs.

	For our purposes, that means:

	1. Assign the file a CDN server (reusing previously assigned
	servers if we've already seen the file).

	2. Strip the dev root pattern (`/static_assets/dev`), and replace
	with the live pattern (`/static_assets/build`)

	3. Tag each file with it's content hash.
	"""
	urls_seen = self.urls_seen
	settings = self.settings

	def replacer( matchobj ):
	url = matchobj.group(0)
	if not url in urls_seen:
	static_file = url.replace( matchobj.group( 1 ), settings[ 'SOURCE_ROOT' ] )
	if self.settings[ 'CDN' ]:
	if not os.path.isfile( static_file ):
	print "WARNING: You've included `%s` in `%s`. It doesn't seem to exist." % ( static_file, source )
	hash = '00000000'
	else:
	hash = hashfile( static_file )
	cdn = settings[ 'CDN_SERVERS' ][ len( urls_seen ) % len( settings[ 'CDN_SERVERS' ] ) ]
	hashed_file = re.sub( r'(.\w+)$', r'.%s\1' % hash, matchobj.group( 2 ) )
	urls_seen[ url ] = "http://%s%s/%s" % ( cdn, settings[ 'LIVE_ROOT_PATTERN' ], hashed_file )
	static_file_dest = os.path.join( settings[ 'OUTPUT_ROOT' ], hashed_file )
	else:
	urls_seen[ url ] = url
	static_file_dest = os.path.join( settings[ 'OUTPUT_ROOT' ], matchobj.group( 2 ) )

	mkdir_p( os.path.dirname( static_file_dest ) )
	shutil.copy( static_file, static_file_dest )

	return urls_seen[ url ]

	return re.sub(
	self.settings[ 'DEV_ROOT_PATTERN' ],
	replacer,
	line)

	def static_combine( self, end_file, to_combine, delimiter="\n/* Begin: %s */\n" ):
	self.log( "* Building `%s`" % os.path.basename( end_file ) )

	if end_file.find( '.css' ) != -1:
	mode = 'CSS'
	else:
	mode = 'JS'

	processed_output = ''
	if mode == 'CSS':
	processed_output = """@charset "UTF-8";\n"""

	for static_file in to_combine:
	if os.path.isfile( static_file ):
	if delimiter:
	processed_output += delimiter % os.path.split( static_file )[ 1 ]
	with open( static_file, 'r' ) as f:
	for line in f:
	processed_output += self.process_urls( line, static_file )

	if processed_output:
	with open( end_file, 'w' ) as combo:
	combo.write( processed_output )

	if self.settings[ 'COMPRESS' ]:
	try:
	command = self.settings[ 'COMPRESSION_COMMAND' ] % end_file
	proc = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE )
	output = proc.communicate()[ 0 ]
	hash = versionhash( output )
	with open( re.sub( r'(\.\w+)$', r'.%s\1' % hash, end_file ), 'w' ) as minified:
	minified.write( output )
	except AttributeError, error:
	raise CommandError("COMPRESSION_COMMAND not set")
	except TypeError, error:
	raise CommandError("No string substitution provided for the input file to be passed to the argument ('cmd %s')")

	def build( self ):
	for filetype in [ 'js', 'css' ]:
	if filetype in self.settings[ 'FILES' ]:
	to_build = self.settings[ 'FILES' ][ filetype ]
	build_root = os.path.join( self.settings[ 'OUTPUT_ROOT' ], filetype )
	mkdir_p( build_root )
	for filename, filelist in to_build.items():
	self.static_combine(
	end_file=os.path.join( build_root, filename ),
	to_combine=[ os.path.join( self.settings[ 'SOURCE_ROOT' ], filetype, f) for f in filelist ]
	)

	def main(argv=None):
	if argv is None:
	argv = sys.argv

	default_root = os.path.dirname(os.path.abspath(__file__))

	parser = OptionParser(usage="Usage: %prog [options]", version="%prog 0.1")
	parser.add_option( "--verbose",
	action="store_true", dest="verbose_mode", default=False,
	help="Verbose mode")

	parser.add_option( "-o", "--output",
	action="store",
	dest="output_root",
	default=os.path.join(default_root, '../build'),
	help="Directory to which output will be written")

	parser.add_option( "-s", "--source",
	action="store",
	dest="source_root",
	default=os.path.join(default_root, "../src"),
	help="Directory from which source files will be read")

	parser.add_option( "-c", "--compress",
	action="store_true",
	dest="compress",
	default=False,
	help="Compress the generated files using the specified compression command")

	parser.add_option( "--cdn",
	action="store_true",
	dest="use_cdn",
	default=False,
	help="Generate CDN URLs for static assets in generated files.")
	(options, args) = parser.parse_args()

	with open( "%s/_config/build.yaml" % options.source_root, 'r' ) as f:
	settings = load( f )

	settings[ 'COMPRESSION_COMMAND' ] = settings[ 'COMPRESSION_COMMAND' ] % default_root

	settings[ 'VERBOSE' ] = options.verbose_mode
	settings[ 'CDN' ] = options.use_cdn
	settings[ 'COMPRESS'] = options.compress
	settings[ 'OUTPUT_ROOT' ] = options.output_root
	settings[ 'SOURCE_ROOT' ] = options.source_root

	builder = StaticAssetBuilder( settings )
	builder.build()


	if __name__ == "__main__":
	sys.exit(main())
	COMPRESSION_COMMAND: 'java -jar "%s/lib/yuicompressor-2.4.2.jar" "%%s"'
	CDN_SERVERS:
	- "pix1.sueddeutsche.de"
	- "pix2.sueddeutsche.de"
	DEV_ROOT_PATTERN: '(/static_assets/dev)/([\-_a-zA-Z0-9\.\/]+)'
	LIVE_ROOT_PATTERN: '/static_assets/build'
	FILES:
	js:
	"grid.js":
	- "copyright.js"
	- "third-party/jquery-1.4.2.js"
	- "third-party/jquery.onAvailable-1.0.js"
	- "third-party/linktracker.js"
	- "third-party/swfobject.js"
	- "sde/sde.js"
	- "advertisement/advertisement.js"
	- "advertisement/google.js"
	- "advertisement/sitestat.js"
	- "advertisement/yahoo.js"
	#
	# JS Utils
	#
	- "utils/utils.js"
	- "utils/config.js"
	- "utils/embedflash.js"
	- "utils/interval.js"
	- "utils/listtoselect.js"
	- "utils/popup.js"
	#
	# Widget Base Classes
	#
	- "widget/widget.js"
	- "widget/dropdowncontrol.js"
	- "widget/lightboxcontrol.js"
	- "widget/overlay.js"
	- "widget/tabcontrol.js"

	#
	# Widget Implementations
	#
	- "widget/bookmarking.js"
	- "widget/dynamiclist.js"
	- "widget/flashelement.js"
	- "widget/login.js"
	- "widget/mailtofriend.js"
	- "widget/poll.js"
	- "widget/projector.js"
	- "widget/tabs.js"
	- "widget/video.js"
	- "widget/weitereangebote.js"
	- "widget/zoomable.js"
	#
	# Initalization
	#
	- "init/siteheader.js"
	- "init/article3.js"
	- "init/homepage.js"
	css:
	"grid.css":
	- "copyright.css"
	- "yui/reset.css"
	- "yui/fonts.css"
	- "decoration.css"
	- "grid.css"
	- "sidebar.css"
	- "site/siteheader.css"
	- "site/sitefooter.css"
	- "colors/colors-ressort.css"
	- "colors/colors-special.css"
	- "article/article.css"
	- "article/header.css"
	- "article/footer.css"
	- "gallery/gallery.css"
	- "gallery/header.css"
	- "gallery/footer.css"
	- "modules/paging.css"
	- "modules/modules.css"
	- "modules/basebox.css"
	- "modules/basebox_columns.css"
	- "modules/basebox_columns_infothek.css"
	- "modules/basebox_columns_sparmeister.css"
	- "modules/basebox_columns_themelist.css"
	- "modules/basebox_comments.css"
	- "modules/basebox_impression.css"
	- "modules/basebox_singlelink.css"
	- "modules/basebox_inlinevideo.css"
	- "modules/basebox_dynamiclist.css"
	- "modules/basebox_focusedteaser.css"
	- "modules/basebox_gallerylist.css"
	- "modules/basebox_imageblock.css"
	- "modules/basebox_imageblock_related.css"
	- "modules/basebox_infothek.css"
	- "modules/basebox_liveticker.css"
	- "modules/basebox_opinion.css"
	- "modules/basebox_szpromo.css"
	- "modules/basebox_tagcloud.css"
	- "modules/basebox_mailtofriend.css"
	- "modules/mehrzumthema.css"
	- "modules/themenbox.css"
	- "modules/inlinemap.css"
	- "modules/bildbanderolle.css"
	- "modules/breakingnewsletter.css"
	- "modules/headslot.css"
	- "modules/headslot_image.css"
	- "modules/bildergaleriebox-stoerer.css"
	- "modules/projector.css"
	- "modules/tabbox.css"
	- "modules/socialbookmarking.css"
	- "modules/servicebox.css"
	- "modules/xlinks.css"
	- "modules/toplisten.css"
	- "modules/poll.css"
	- "modules/tabbox_finance.css"
	- "modules/teaserlist.css"
	- "modules/teaserlist_breakingnews.css"
	- "modules/teaserlist_metadata.css"
	- "modules/teaserlist_paging.css"
	- "modules/teaserlist_topteaser.css"
	- "modules/themehighlights.css"
	# Ressortblock, _then_ stockchart, _then_ basebox_stockchart.
	- "modules/ressortblock.css"
	- "modules/stockchart.css"
	- "modules/basebox_stockchart.css"
	- "modules/hp_sonderthemen.css"
	# Whisper, _then_ basebox_whisper.
	- "modules/whisper.css"
	- "modules/basebox_whisper.css"
	#
	# Advertisement Styles. These are my favourites.
	#
	- "advertisement/yahoo.css"
	- "advertisement/google.css"
	- "advertisement/ivw.css"