xinomilo/find-https-debian-archives.py

## find-https-debian-archives.py
#!/usr/bin/python

import urllib2
import re
import ssl
import sys

# # find generic mirrors
mirrors = urllib2.urlopen('https://www.debian.org/mirror/list')
https = []
for line in mirrors.readlines():
    m = re.match('.*<td valign="top"><a rel="nofollow" href="http(.*)">.*', line)
    if m:
        url = 'https' + m.group(1)
        print 'trying: ',
        print url,
        print '...',
        sys.stdout.flush()
        try:
            response=urllib2.urlopen(url, timeout=1)
            https.append(url)
            print 'success!'
        except urllib2.URLError as err:
            print 'fail!'
        except ssl.SSLError as err:
            print 'bad SSL!'
 	except ssl.CertificateError as err:
 	    print 'Certificate Error!'


# print 'HTTPS apt repos:'
#for url in https:
#    print url


# # find security mirrors
mirrors = urllib2.urlopen('https://www.debian.org/mirror/list-full')
securitys = []
for line in mirrors.readlines():
    m = re.match('.*</tt><br>Security updates over HTTP: <tt><a rel="nofollow" href="http(.*)">.*/debian-security/</a>.*', line)
    if m:
        url = 'https' + m.group(1)
        print 'trying: ',
        print url,
        print '...',
        sys.stdout.flush()
        try:
            response=urllib2.urlopen(url, timeout=1)
            securitys.append(url)
            print 'success!'
        except urllib2.URLError as err:
            print 'fail!'
        except ssl.SSLError as err:
            print 'bad SSL!'
 	except ssl.CertificateError as err:
 	    print 'Certificate Error!'

# print 'HTTPS security repos:'
# for url in securitys:
#     print url


# now find the backports mirrors
mirrors = urllib2.urlopen('http://backports-master.debian.org/Mirrors/')
backports = []
for line in mirrors.readlines():
#<td><a href="http://be.mirror.eurid.eu/debian-backports/">/debian-backports/</a>
    m = re.match('.*<td><a href="http(.*)">.*/debian-backports/</a>.*', line)
    if m:
        url = 'https' + m.group(1)
        print 'trying: ',
        print url,
        print '...',
        sys.stdout.flush()
        try:
            response=urllib2.urlopen(url, timeout=1)
            backports.append(url)
            print 'success!'
        except urllib2.URLError as err:
            print 'fail!'
        except ssl.SSLError as err:
            print 'bad SSL!'
 	except ssl.CertificateError as err:
 	    print 'Certificate Error!'

#print 'HTTPS backports repos:'
#for url in backports:
#    print url


# now find the CD image mirrors
mirrors = urllib2.urlopen('https://www.debian.org/CD/http-ftp/')
cds = []
for line in mirrors.readlines():
# <a rel="nofollow" href="http://mirror.easyspeedy.com/debian-cd/">HTTP</a></li>
    m = re.match('.*<a rel="nofollow" href="http(:.*)">HTTP</a></li>.*', line)
    if m:
        url = 'https' + m.group(1)
        print 'trying: ',
        print url,
        print '...',
        sys.stdout.flush()
        try:
            response=urllib2.urlopen(url, timeout=1)
            cds.append(url)
            print 'success!'
        except urllib2.URLError as err:
            print 'fail!'
        except ssl.SSLError as err:
            print 'bad SSL!'
 	except ssl.CertificateError as err:
 	    print 'Certificate Error!'

print 'HTTPS CD image repos:'
for url in cds:
    print url


# now write everything to a file
f = open('/tmp/https-debian-archives.txt', 'w')

f.write('HTTPS apt repos\n')
f.write('---------------\n')
for url in https:
    f.write(url + '\n')

f.write('\n\nHTTPS security repos\n')
f.write('---------------\n')
for url in securitys:
    f.write(url + '\n')

f.write('\n\nHTTPS backports repos\n')
f.write('--------------------\n')
for url in backports:
    f.write(url + '\n')

f.write('\n\nHTTPS CD image repos\n')
f.write('--------------------\n')
for url in cds:
    f.write(url + '\n')

f.close()
	#!/usr/bin/python

	import urllib2
	import re
	import ssl
	import sys

	# # find generic mirrors
	mirrors = urllib2.urlopen('https://www.debian.org/mirror/list')
	https = []
	for line in mirrors.readlines():
	m = re.match('.<td valign="top"><a rel="nofollow" href="http(.)">.*', line)
	if m:
	url = 'https' + m.group(1)
	print 'trying: ',
	print url,
	print '...',
	sys.stdout.flush()
	try:
	response=urllib2.urlopen(url, timeout=1)
	https.append(url)
	print 'success!'
	except urllib2.URLError as err:
	print 'fail!'
	except ssl.SSLError as err:
	print 'bad SSL!'
	except ssl.CertificateError as err:
	print 'Certificate Error!'


	# print 'HTTPS apt repos:'
	#for url in https:
	# print url


	# # find security mirrors
	mirrors = urllib2.urlopen('https://www.debian.org/mirror/list-full')
	securitys = []
	for line in mirrors.readlines():
	m = re.match('.</tt><br>Security updates over HTTP: <tt><a rel="nofollow" href="http(.)">./debian-security/</a>.', line)
	if m:
	url = 'https' + m.group(1)
	print 'trying: ',
	print url,
	print '...',
	sys.stdout.flush()
	try:
	response=urllib2.urlopen(url, timeout=1)
	securitys.append(url)
	print 'success!'
	except urllib2.URLError as err:
	print 'fail!'
	except ssl.SSLError as err:
	print 'bad SSL!'
	except ssl.CertificateError as err:
	print 'Certificate Error!'

	# print 'HTTPS security repos:'
	# for url in securitys:
	# print url


	# now find the backports mirrors
	mirrors = urllib2.urlopen('http://backports-master.debian.org/Mirrors/')
	backports = []
	for line in mirrors.readlines():
	#<td><a href="http://be.mirror.eurid.eu/debian-backports/">/debian-backports/</a>
	m = re.match('.<td><a href="http(.)">./debian-backports/</a>.', line)
	if m:
	url = 'https' + m.group(1)
	print 'trying: ',
	print url,
	print '...',
	sys.stdout.flush()
	try:
	response=urllib2.urlopen(url, timeout=1)
	backports.append(url)
	print 'success!'
	except urllib2.URLError as err:
	print 'fail!'
	except ssl.SSLError as err:
	print 'bad SSL!'
	except ssl.CertificateError as err:
	print 'Certificate Error!'

	#print 'HTTPS backports repos:'
	#for url in backports:
	# print url


	# now find the CD image mirrors
	mirrors = urllib2.urlopen('https://www.debian.org/CD/http-ftp/')
	cds = []
	for line in mirrors.readlines():
	# <a rel="nofollow" href="http://mirror.easyspeedy.com/debian-cd/">HTTP</a></li>
	m = re.match('.<a rel="nofollow" href="http(:.)">HTTP</a></li>.*', line)
	if m:
	url = 'https' + m.group(1)
	print 'trying: ',
	print url,
	print '...',
	sys.stdout.flush()
	try:
	response=urllib2.urlopen(url, timeout=1)
	cds.append(url)
	print 'success!'
	except urllib2.URLError as err:
	print 'fail!'
	except ssl.SSLError as err:
	print 'bad SSL!'
	except ssl.CertificateError as err:
	print 'Certificate Error!'

	print 'HTTPS CD image repos:'
	for url in cds:
	print url


	# now write everything to a file
	f = open('/tmp/https-debian-archives.txt', 'w')

	f.write('HTTPS apt repos\n')
	f.write('---------------\n')
	for url in https:
	f.write(url + '\n')

	f.write('\n\nHTTPS security repos\n')
	f.write('---------------\n')
	for url in securitys:
	f.write(url + '\n')

	f.write('\n\nHTTPS backports repos\n')
	f.write('--------------------\n')
	for url in backports:
	f.write(url + '\n')

	f.write('\n\nHTTPS CD image repos\n')
	f.write('--------------------\n')
	for url in cds:
	f.write(url + '\n')

	f.close()