Get SHA1 hashes from archive.org metadata

extract-archive-org-hashes.py

#!/usr/bin/env python

###############################
# Extract SHA1 file hashes from archive.org metadata files
#
# Usage: place archive.org {item_name}_files.xml in ./xml/
# Run ./extract-hashes.py
# sha1 files get written to ./sha1
###############################

import xml.etree.ElementTree as ET
import os
import glob

files = glob.glob('xmls/*.xml')

blacklist = [".xml", ".torrent", ".jpg", ".png", ".sqlite"]

for filename in files:
  f = open('sha1/' + os.path.basename(filename).replace('.xml', '.sha1'), 'w')
  root = ET.parse(filename).getroot()

  accumulator = 0

  for type_tag in root.findall('file'):
    file_name = type_tag.get('name')
    if os.path.splitext(file_name)[-1].lower() not in map(str.lower, blacklist):
        sha1 = type_tag.findall('sha1')
        size = type_tag.findall('size')
        if len(sha1) > 0:
          f.write('%s  %s\n' % (sha1[0].text, os.path.basename(file_name)))

          file_size = int(size[0].text)
          accumulator = accumulator + int(file_size)

  gb = round(accumulator / 1000 / 1000 / 1000)
  print('%s %s Gb' %(filename, gb))
  f.close()