remove errors from nginx cache

clean_cache.py

#!/usr/bin/env python3
# apt install python3-tqdm

import os
import sys
from glob import iglob
from tqdm import tqdm

root = '/var/cache/nginx/octodon'

HEADER_SIZE = 2048

def main():
    print("Listing files... ", end="")
    sys.stdout.flush()
    files = iglob(os.path.join(root, '**'), recursive=True)
    files = (f for f in files if not f.startswith(os.path.join(root, 'temp/')))
    files = (f for f in files if os.path.isfile(f))
    files = list(files)
    print(len(files))

    bar = tqdm(files)
    for fp in bar:
        try:
            with open(fp, 'rb') as f: 
                header = f.read(HEADER_SIZE)[0x91:].split(b'\r\n\r\n', 1)[0]
                headers = header.decode('utf-8').split('\n')
                key = headers[0]
                http = headers[1]

                assert http.startswith('HTTP/1.'), http

                code = int(http.split(' ')[1])
                if code >= 400:
                    bar.write("Deleting HTTP {} {}".format(code, fp))
                    os.unlink(fp)
        except FileNotFoundError:
            pass


main()