Osi Emoekpere osiloke

## gist:1138798
from scrapy import log
from scrapy.item import Item
from scrapy.http import Request
from scrapy.contrib.spiders import XMLFeedSpider


def NextURL():
    """
    Generate a list of URLs to crawl. You can query a database or come up with some other means
    Note that if you generate URLs to crawl from a scraped URL then you're better of using a

## extractors.py
import re

from scrapy.link import Link
from urlparse import urljoin
from BeautifulSoup import BeautifulSoup

class SoupLinkExtractor(object):
    def __init__(self, *args, **kwargs):
        super(SoupLinkExtractor, self).__init__()
        allow_re = kwargs.get('allow', None)

## transfer-network-config
Some experiments, which worked but were eventually not used.
See http://www.greenhills.co.uk/2011/06/10/lxc.html

Now, the next big milestone is to get networking working.
In the config, we told lxc-create to use br0 as the link,
and inside the container we have eth0:

    root@thunder:/# brctl show br0
    bridge name	bridge id		STP enabled	interfaces
    br0		8000.962e6bb72480	no		vethP70LNx

## lxc.rb
package 'debootstrap'
package 'apt-cacher-ng'
package 'inotify-tools'

include_recipe 'lxc::manage'
include_recipe 'lxc::network_bridge'
include_recipe 'lxc::dns'

host = node[:container]

## native.scroll.html
<!DOCTYPE html>
<html>
<!-- NOTES
Demo - uses IOS native scroll available in IOS5, but sidestep the document-bounce behavior. Tested on iPad 1, using IOS 5.1.

This approach uses a set of 3 nested divs.
OuterDiv -- fixed height, width, overflow-scrolling: touch;
MiddleDiv -- fixed height width, overflow-scrolling: touch; fits inside OuterDiv
InnerDiv -- fixed height width, bigger than MiddleDiv, so it kicks in the overflow behavior

## default.rb
#
# Cookbook Name:: s3_backups
# Recipe:: default
# Author:: Todd Zusman (toddzusman.com)
#

execute "apt-get update" do
  action :nothing
end

## nginx_cors_s3_upload_proxy_full
# DO NOT RESPOND TO REQUESTS OTHER THAN yourdomain.com
server {
    listen       80  default;
    server_name  _;
    return       444;
}

# FILE UPLOADS
server {
        listen   80;

## libdnet_python.rb
require 'formula'

class LibdnetPython <Formula
  depends_on 'libdnet'
  url 'http://libdnet.googlecode.com/files/libdnet-1.12.tgz'
  homepage 'http://code.google.com/p/libdnet/'
  sha1 '71302be302e84fc19b559e811951b5d600d976f8'

  def install
    ENV["CFLAGS"] = "-O3 -w -pipe"

## ActionBarExample.java
// Copyright (C) 2013 Polychrom Pty Ltd
//
// This program is licensed under the 3-clause "Modified" BSD license,
// see LICENSE file for full definition.

package com.polychrom.examples;

import java.util.concurrent.ExecutorService;

import android.app.Activity;

## txredns.py
# Python/Twisted/Redis backed DNS server - resolves from NAME to IP addrs
# fallback to google or any other DNS server to resolv domains not present on Redis
# to set a new domain on redis, just issue a SET domain.tld ip_addr
# run with twistd -ny txredns.tac
# gleicon 2011

from twisted.names import dns, server, client, cache
from twisted.application import service, internet
from twisted.internet import defer
from twisted.python import log
	from scrapy import log
	from scrapy.item import Item
	from scrapy.http import Request
	from scrapy.contrib.spiders import XMLFeedSpider


	def NextURL():
	"""
	Generate a list of URLs to crawl. You can query a database or come up with some other means
	Note that if you generate URLs to crawl from a scraped URL then you're better of using a
	import re

	from scrapy.link import Link
	from urlparse import urljoin
	from BeautifulSoup import BeautifulSoup

	class SoupLinkExtractor(object):
	def __init__(self, args, *kwargs):
	super(SoupLinkExtractor, self).__init__()
	allow_re = kwargs.get('allow', None)
	Some experiments, which worked but were eventually not used.
	See http://www.greenhills.co.uk/2011/06/10/lxc.html

	Now, the next big milestone is to get networking working.
	In the config, we told lxc-create to use br0 as the link,
	and inside the container we have eth0:

	root@thunder:/# brctl show br0
	bridge name bridge id STP enabled interfaces
	br0 8000.962e6bb72480 no vethP70LNx
	package 'debootstrap'
	package 'apt-cacher-ng'
	package 'inotify-tools'

	include_recipe 'lxc::manage'
	include_recipe 'lxc::network_bridge'
	include_recipe 'lxc::dns'

	host = node[:container]
	<!DOCTYPE html>
	<html>
	<!-- NOTES
	Demo - uses IOS native scroll available in IOS5, but sidestep the document-bounce behavior. Tested on iPad 1, using IOS 5.1.

	This approach uses a set of 3 nested divs.
	OuterDiv -- fixed height, width, overflow-scrolling: touch;
	MiddleDiv -- fixed height width, overflow-scrolling: touch; fits inside OuterDiv
	InnerDiv -- fixed height width, bigger than MiddleDiv, so it kicks in the overflow behavior
	#
	# Cookbook Name:: s3_backups
	# Recipe:: default
	# Author:: Todd Zusman (toddzusman.com)
	#

	execute "apt-get update" do
	action :nothing
	end
	# DO NOT RESPOND TO REQUESTS OTHER THAN yourdomain.com
	server {
	listen 80 default;
	server_name _;
	return 444;
	}

	# FILE UPLOADS
	server {
	listen 80;
	require 'formula'

	class LibdnetPython <Formula
	depends_on 'libdnet'
	url 'http://libdnet.googlecode.com/files/libdnet-1.12.tgz'
	homepage 'http://code.google.com/p/libdnet/'
	sha1 '71302be302e84fc19b559e811951b5d600d976f8'

	def install
	ENV["CFLAGS"] = "-O3 -w -pipe"
	// Copyright (C) 2013 Polychrom Pty Ltd
	//
	// This program is licensed under the 3-clause "Modified" BSD license,
	// see LICENSE file for full definition.

	package com.polychrom.examples;

	import java.util.concurrent.ExecutorService;

	import android.app.Activity;
	# Python/Twisted/Redis backed DNS server - resolves from NAME to IP addrs
	# fallback to google or any other DNS server to resolv domains not present on Redis
	# to set a new domain on redis, just issue a SET domain.tld ip_addr
	# run with twistd -ny txredns.tac
	# gleicon 2011

	from twisted.names import dns, server, client, cache
	from twisted.application import service, internet
	from twisted.internet import defer
	from twisted.python import log