amitdash

# Change YOUR_TOKEN to your prerender token and uncomment that line if you want to cache urls and view crawl stats
# Change example.com (server_name) to your website url
# Change /path/to/your/root to the correct value

server {
    listen 80;
    server_name example.com;
 
    root   /path/to/your/root;
    index  index.html;

amitdash

worker_processes 2;

error_log /var/log/nginx/error.log;
pid /var/run/nginx.pid;

events {
    worker_connections 1024;
    use epoll;
}

amitdash

# Author = Nikhil Venkat Sonti
# email = nikhilsv92@gmail.com
# github ID = shadowfax92
import sys
from xml.dom.minidom import _get_StringIO
from lxml import html
import requests
import os
import re
import time

amitdash

import argparse
import re
from multiprocessing.pool import ThreadPool as Pool
import requests
import bs4

root_url = 'http://pyvideo.org'
index_url = root_url + '/category/50/pycon-us-2014'

amitdash

worker_processes 2;

error_log /var/log/nginx/error.log;
pid /var/run/nginx.pid;

events {
    worker_connections 1024;
    use epoll;
}

amitdash

from datetime import datetime
from pymongo.connection import Connection

import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web

from tornado.options import define, options