cujanovic

#!/bin/sh
#
#      `7MN.   `7MF'       
# __,    MMN.    M         
#`7MM    M YMb   M  pd""b. 
#  MM    M  `MN. M (O)  `8b
#  MM    M   `MM.M      ,89
#  MM    M     YMM    ""Yb.
#.JMML..JML.    YM       88
#                  (O)  .M'

cujanovic

import requests
import re
import sys
from multiprocessing.dummy import Pool


def robots(host):
    r = requests.get(
        'https://web.archive.org/cdx/search/cdx\
        ?url=%s/robots.txt&output=json&fl=timestamp,original&filter=statuscode:200&collapse=digest' % host)

cujanovic

user www-data;

http {

  ##
  # Basic Settings
  ##

  sendfile on;
  tcp_nopush on;