Link rot in kulturarvsdata

checkkulturarv Wikipedia links.py

''' User a query from Quarry that generates 164 143 links
the result is error in 7602 unique URLs
* Wikipedia diskussion https://sv.wikipedia.org/wiki/Malldiskussion:BBR-l%C3%A4nk#BeBR_410-poster_tillbaka
* Notebook https://github.com/salgo60/open-data-examples/blob/master/Icke%20fungerande%20RA%C3%84%20l%C3%A4nkar.ipynb
'''

import urllib3
import sys,json
http = urllib3.PoolManager()

def get_file():
    filename = "/Users/magnus/Documents/GitHub/PythonDataScienceHandbook/notebooks/Ksamsok20200629.json"
    with open(filename) as jsonfile:
        data = json.load(jsonfile)
        return data
    return False

def write_file(NonWorkingURL):
    with open('nonWorkingURLS.txt', 'w') as fw:
        for x in NonWorkingURL:
            fw.writelines(x)

''' Check if URLS is ok'''
def check(url):
    r = http.request('GET', url)
    if r.status != 200:
        print("Status: ",r.status, " \t", url)
        return False
    return True

ok = 0
notok = 0
NonWorkingURL = set()
results = get_file()
for r in results["rows"]:
    url = (r[0])
    if check(url):
        ok +=1
    else:
        notok +=1
        NonWorkingURL.add(url)

print("Ok: ",ok,"\tNot ok", notok)
write_file(NonWorkingURL)

164143
Result: Ok: 156539 Not ok 7604

See

• file list https://gist.github.com/salgo60/4f2fa0370a5541712fd274e5b9e84b26
• Jupyter Notebook with the result showing non 200 status requests

new test 2020-11-20
160548
OK: 154158 not ok 6390