rsms.me/stuff

_000_readme.md

setup

mkdir -p stuff/_web
cd stuff
wget https://gist.githubusercontent.com/rsms/9925894f0fff266d392bc149c648e535/raw/_gen.py
wget https://gist.githubusercontent.com/rsms/9925894f0fff266d392bc149c648e535/raw/_index.template.html
cat << END > _web/aws-credentials.conf
[default]
aws_access_key_id = YOURKEY
aws_secret_access_key = YOURSECRET
END
yourpackagemanager install imagemagick ffmpeg sox gnuplot
open https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
# ^ install AWS cli tools

usage

Put stuff in your "stuff" directory and run python3 _gen.py

_gen.py

#!/usr/bin/env python3
#
# Prerequisites:
#   brew install imagemagick ffmpeg sox gnuplot
#
#   cd _web
#   wget https://downloads.sourceforge.net/project/gnuplot/gnuplot/5.4.6/gnuplot-5.4.6.tar.gz
#   tar xf gnuplot-5.4.6.tar.gz
#   cd gnuplot-5.4.6
#   ./configure && make -j$(nproc)
#   ./src/gnuplot --help
#
#   https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
#
import os.path as path
from urllib.parse import quote as urlescape
from pathlib import Path
import os, sys, string, html, mimetypes, io, struct, subprocess, time
import base64, hashlib, json, PIL.Image, math
from multiprocessing.pool import ThreadPool
from tempfile import TemporaryDirectory
from os.path import basename
from os.path import join as pjoin
from datetime import datetime

SELF_SCRIPT_NAME = basename(__file__)
os.chdir(path.dirname(path.realpath(__file__)))
ROOT_DIR = os.getcwd()
print(f"cd {ROOT_DIR}")


IMG_MAX_SIZE = 128
THUMB_SIZE = int(IMG_MAX_SIZE * 2)
THUMBS_DIR = '_web/thumbs'
MIRROR_DIR = '_web/mirror'
# GNUPLOT = '_web/gnuplot-5.4.6/src/gnuplot'
GNUPLOT = 'gnuplot'

INDEX_TPL = string.Template(Path("_index.template.html").read_text())

DEFAULT_ITEM_TPL = string.Template('''
<a href="$path" class="item default$css_class" id="$id" $style_attr title="$name ($mtime)" data-mtime="$mtime_id"><span class=name>$name</span><span class=mtime>$mtime</span></a>
'''.strip())

IMG_ITEM_TPL = string.Template('''
<a href="$path" class="item img$css_class" style="background-image:url($thumb_path)" id="$id" title="$name ($mtime)" data-mtime="$mtime_id"><div class="ani-overlay" style="background-image:url($thumb_gif)"></div><span class=name>$name</span><span class=mtime>$mtime</span></a>
'''.strip())

DIR_ITEM_TPL = string.Template('''
<a href="$path" class="item dir" id=$id><span class=name>$name</span></a>
'''.strip())


pool = ThreadPool(os.cpu_count())
thumbs = []


class UnknownImageFormat(Exception):
  pass


def _image_size(file):
  # This function is licensed as follows:
  #
  # The MIT License (MIT)
  #
  # Copyright (c) 2013 Paulo Scardine
  #
  # Permission is hereby granted, free of charge, to any person obtaining a copy of
  # this software and associated documentation files (the "Software"), to deal in
  # the Software without restriction, including without limitation the rights to
  # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
  # the Software, and to permit persons to whom the Software is furnished to do so,
  # subject to the following conditions:
  #
  # The above copyright notice and this permission notice shall be included in all
  # copies or substantial portions of the Software.
  #
  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
  # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
  # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  size = path.getsize(file)
  with io.open(file, "rb") as f:
    width = -1
    height = -1
    data = f.read(26)
    msg = " raised while trying to decode as JPEG."
    if (size >= 10) and data[:6] in (b'GIF87a', b'GIF89a'):
      # GIFs
      imgtype = 'GIF'
      w, h = struct.unpack("<HH", data[6:10])
      width = int(w)
      height = int(h)
    elif ((size >= 24) and data.startswith(b'\211PNG\r\n\032\n')
        and (data[12:16] == b'IHDR')):
      # PNGs
      imgtype = 'PNG'
      w, h = struct.unpack(">LL", data[16:24])
      width = int(w)
      height = int(h)
    elif (size >= 16) and data.startswith(b'\211PNG\r\n\032\n'):
      # older PNGs
      imgtype = 'PNG'
      w, h = struct.unpack(">LL", data[8:16])
      width = int(w)
      height = int(h)
    elif (size >= 2) and data.startswith(b'\377\330'):
      # JPEG
      imgtype = 'JPEG'
      f.seek(0)
      f.read(2)
      b = f.read(1)
      try:
        while (b and ord(b) != 0xDA):
          while (ord(b) != 0xFF):
            b = f.read(1)
          while (ord(b) == 0xFF):
            b = f.read(1)
          if (ord(b) >= 0xC0 and ord(b) <= 0xC3):
            f.read(3)
            h, w = struct.unpack(">HH", f.read(4))
            break
          else:
            f.read(
              int(struct.unpack(">H", f.read(2))[0]) - 2)
          b = f.read(1)
        width = int(w)
        height = int(h)
      except struct.error:
        raise UnknownImageFormat("StructError" + msg)
      except ValueError:
        raise UnknownImageFormat("ValueError" + msg)
      except Exception as e:
        raise UnknownImageFormat(e.__class__.__name__ + msg)
    elif (size >= 26) and data.startswith(b'BM'):
      # BMP
      imgtype = 'BMP'
      headersize = struct.unpack("<I", data[14:18])[0]
      if headersize == 12:
        w, h = struct.unpack("<HH", data[18:22])
        width = int(w)
        height = int(h)
      elif headersize >= 40:
        w, h = struct.unpack("<ii", data[18:26])
        width = int(w)
        # as h is negative when stored upside down
        height = abs(int(h))
      else:
        raise UnknownImageFormat(
          "Unkown DIB header size:" +
          str(headersize))
    elif (size >= 8) and data[:4] in (b"II\052\000", b"MM\000\052"):
      # Standard TIFF, big- or little-endian
      # BigTIFF and other different but TIFF-like formats are not
      # supported currently
      imgtype = 'TIFF'
      byteOrder = data[:2]
      boChar = ">" if byteOrder == "MM" else "<"
      # maps TIFF type id to size (in bytes)
      # and python format char for struct
      tiffTypes = {
        1: (1, boChar + "B"),  # BYTE
        2: (1, boChar + "c"),  # ASCII
        3: (2, boChar + "H"),  # SHORT
        4: (4, boChar + "L"),  # LONG
        5: (8, boChar + "LL"),  # RATIONAL
        6: (1, boChar + "b"),  # SBYTE
        7: (1, boChar + "c"),  # UNDEFINED
        8: (2, boChar + "h"),  # SSHORT
        9: (4, boChar + "l"),  # SLONG
        10: (8, boChar + "ll"),  # SRATIONAL
        11: (4, boChar + "f"),  # FLOAT
        12: (8, boChar + "d")   # DOUBLE
      }
      ifdOffset = struct.unpack(boChar + "L", data[4:8])[0]
      try:
        countSize = 2
        f.seek(ifdOffset)
        ec = f.read(countSize)
        ifdEntryCount = struct.unpack(boChar + "H", ec)[0]
        # 2 bytes: TagId + 2 bytes: type + 4 bytes: count of values + 4
        # bytes: value offset
        ifdEntrySize = 12
        for i in range(ifdEntryCount):
          entryOffset = ifdOffset + countSize + i * ifdEntrySize
          f.seek(entryOffset)
          tag = f.read(2)
          tag = struct.unpack(boChar + "H", tag)[0]
          if(tag == 256 or tag == 257):
            # if type indicates that value fits into 4 bytes, value
            # offset is not an offset but value itself
            type = f.read(2)
            type = struct.unpack(boChar + "H", type)[0]
            if type not in tiffTypes:
              raise UnknownImageFormat(
                "Unkown TIFF field type:" +
                str(type))
            typeSize = tiffTypes[type][0]
            typeChar = tiffTypes[type][1]
            f.seek(entryOffset + 8)
            value = f.read(typeSize)
            value = int(struct.unpack(typeChar, value)[0])
            if tag == 256:
              width = value
            else:
              height = value
          if width > -1 and height > -1:
            break
      except Exception as e:
        raise UnknownImageFormat(str(e))
    elif size >= 2:
        # see http://en.wikipedia.org/wiki/ICO_(file_format)
      imgtype = 'ICO'
      f.seek(0)
      reserved = f.read(2)
      if 0 != struct.unpack("<H", reserved)[0]:
        raise UnknownImageFormat()
      format = f.read(2)
      assert 1 == struct.unpack("<H", format)[0]
      num = f.read(2)
      num = struct.unpack("<H", num)[0]
      if num > 1:
        import warnings
        warnings.warn("ICO File contains more than one image")
      # http://msdn.microsoft.com/en-us/library/ms997538.aspx
      w = f.read(1)
      h = f.read(1)
      width = ord(w)
      height = ord(h)
    else:
      raise UnknownImageFormat()

    # return (width, height, imgtype, size)
    return (width, height)


def image_size_identify(file):
  # print(f"_image_size({file}) PIL failed, trying identify")
  p = subprocess.run(['identify', '-format', r'%w %h;', file], capture_output=True)
  try:
    # note re ';': some file formats like .psd have many sizes
    t = [int(s) for s in p.stdout.split(b';', 1)[0].split(b' ')]
    return (t[0], t[1])
  except:
    return (0, 0)

def image_size(file):
  try:
    return _image_size(file)
  except UnknownImageFormat:
    # print(f"_image_size({file}) failed, trying PIL")
    try:
      img = PIL.Image.open(file)
      bbox = img.getbbox()
      w = bbox[2] - bbox[0]
      h = bbox[3] - bbox[1]
      return (w, h)
    except:
      return image_size_identify(file)


def thumb_filename(filename, size=THUMB_SIZE, ext=''):
  with open(filename, "rb") as f:
    f.seek(0, os.SEEK_END)
    filesize = f.tell()
    f.seek(0, os.SEEK_SET)
    hash = hashlib.file_digest(f, "sha1")
  hash = base64.b64encode(hash.digest(), b'+_').strip(b'=').decode('utf8')
  filename_ext = path.splitext(filename)[1]
  thumbfile = filename
  if filesize > 10*1024 or \
     filename_ext in ('.gif', '.psd') or \
     (ext != '' and ext != filename_ext):
    thumbfile = pjoin(THUMBS_DIR, hash)
    if ext == '':
      ext = '.png' if size < THUMB_SIZE or filename_ext == '.pdf' else '.jpg'
      if filename_ext == '.png':
        # force use of png for images with transparency
        # print(f"checking for transparency in {filename}...")
        p = subprocess.run([
          'convert', filename, '-format', '%[opaque]', 'info:',
        ], capture_output=True)
        if p.returncode == 0 and p.stdout.find(b'False') != -1:
          ext = '.png'
    thumbfile += ext
  return (thumbfile, hash)


def gen_thumb_img(filename, thumbfile, w=0, h=0):
  print(f"gen_thumb_img {filename} -> {thumbfile}")
  ext = path.splitext(filename)[1]
  # note re "[0]": only extract first/primary image from files like gif, pdf, psd etc.
  args = ['convert']
  # args += ['-verbose']
  if ext == '.pdf':
    if min(w,h) == 0:
      w, h = image_size_identify(filename)
    if w != 0 and min(w,h) < THUMB_SIZE:
      args += [ '-density', str(int(72 * (THUMB_SIZE/min(w,h)))) ]
  args += [
    filename + "[0]", '-strip',
    '-colorspace','sRGB','-type','truecoloralpha', # make greyscale images color
    # '-define','png:color-type=2', # make greyscale images color
  ]
  if ext == '.pdf':
    # no special args
    pass
  elif w == 0 or h == 0 or w > THUMB_SIZE or h > THUMB_SIZE:
    args += [
      '-resize','x1024',
      '-sharpen','2x2',
      '-resize', 'x' + str(THUMB_SIZE),
      '-quality','60',
    ]
  elif w <= THUMB_SIZE/1.5 or h <= THUMB_SIZE/1.5:
    args += [
      '-interpolate', 'nearest-neighbor',
      '-filter', 'point',
      '-resize', 'x' + str(THUMB_SIZE),
      '-quality','90',
    ]
  else:
    args += [
      '-quality','60',
    ]
  args += [ thumbfile ]
  subprocess.run(args, check=True)


def gen_video_thumb_gif(filename, thumbfile_gif, thumbfile_png):
  paletteuse = '[s1][p]paletteuse'
  paletteuse += '=dither=bayer'
  scale_filter = f'scale={THUMB_SIZE}:-1:flags=lanczos'
  common_args = [
    'ffmpeg', '-nostdin', '-loglevel','warning', '-y', '-i', filename,
  ]
  fps = 15

  # gif
  print(f"gen_video_thumb_gif {filename} -> {thumbfile_gif}")
  filters = [
    f'fps={fps}',
    scale_filter,
    f'split[s0][s1];[s0]palettegen=stats_mode=full:max_colors=32[p];{paletteuse}',
  ]
  interval = 5
  nframes = int(fps/3)
  limit_filters = [
    f"select='if(not(floor(mod(t,{interval})))*lt(ld(1),1),st(1,1)+st(2,n)+st(3,t));if(eq(ld(1),1)*lt(n,ld(2)+{nframes}),1,if(trunc(t-ld(3)),st(1,0)))'",
    f"setpts=N/{fps}/TB",
  ]
  # limit_filters = [ "select=between(mod(n\\,120)\\,0\\,7),setpts=N/24/TB" ]
  is_short = False
  p = subprocess.run(common_args + [
    '-ss','00:00:01', '-t','30', # start at 1sec into video, stop after 30 seconds
    '-vf', ','.join(filters + limit_filters),
    '-loop', '0',
    thumbfile_gif], check=True, capture_output=True)
  if p.stderr.find(b'Output file is empty') != -1:
    print(f"gen_video_thumb_gif {filename} -> {thumbfile_gif} (full)")
    is_short = True
    p = subprocess.run(common_args + [
      '-vf', ','.join(filters),
      '-loop', '0',
      thumbfile_gif], check=True)

  # single-frame png
  print(f"gen_video_thumb_gif {filename} -> {thumbfile_png}")
  times = ['00:00:03', '00:00:02', '00:00:01', '']
  if is_short:
    times = ['']
  for ss in times:
    p = subprocess.run(common_args + ([] if ss == '' else ['-ss',ss]) + [
      '-update','0', '-frames:v','1',
      '-vf', scale_filter,
      thumbfile_png], check=True, capture_output=True)
    if p.stderr.find(b'Output file is empty') == -1:
      break


def audio_duration_secs(filename):
  p = subprocess.run(['sox', '--info', '-D', filename], capture_output=True)
  # print(f"soxi {filename}: returncode={p.returncode} stderr={p.stderr}")
  if p.returncode != 0:
    print(f"skip audio file {filename}: {p.stderr.decode('utf8')}")
  return float(p.stdout) if p.returncode == 0 else 0.0


def gen_audio_thumb(filename, thumbfile, duration_secs):
  with TemporaryDirectory(prefix='stuff-audio-') as tmpdir:
    datfile1 = pjoin(tmpdir, 'audio1.dat')
    datfile = pjoin(tmpdir, 'audio.dat')
    gpifile = pjoin(tmpdir, 'audio.gpi')

    print(f"gen_audio_thumb {filename} -> {thumbfile} ({duration_secs} sec)")

    trim_start = min(int(duration_secs / 8), 4) # seconds
    trim_len   = min(60, max(1, int(duration_secs / 4))) # seconds
    #print(f"trim_start={trim_start} trim_len={trim_len}")
    subprocess.run([
      'sox', filename, datfile1,
      'trim', str(trim_start), f'00:{trim_len}',
    ], check=True)
    with open(datfile, 'wb') as wf:
      with open(datfile1, 'rb') as rf:
        for line in rf:
          if not line.startswith(b';'):
            break
        wf.write(rf.read())

    h = int(THUMB_SIZE / 2)

    gpitext = f'''
      # set output format and size
      set terminal pngcairo size {THUMB_SIZE},{h} transparent truecolor

      #set output file
      set output "{thumbfile}"

      # set y range
      set yr [-1:1]

      # we want just the data
      unset key
      unset tics
      unset border
      set lmargin 0
      set rmargin 0
      set tmargin 0
      set bmargin 0

      # draw rectangle to change background color
      # set obj 1 rectangle behind from screen 0,0 to screen 1,1
      # set obj 1 fillstyle solid 1.0 fillcolor rgbcolor "#222222"

      set style line 1  linetype 2  linecolor 'blue'  linewidth 2

      # draw data with foreground color
      plot "{datfile}" every 32 with lines lt rgb '#44000000',\\
                    '' every 64 with lines lt rgb '#66000000',\\
                    '' every 128 with lines lt rgb '#77000000',\\
                    '' every 256 with l smooth mcsplines lw 4 lc '#aa000000'
      '''.strip().replace('        ', '') + '\n'
    Path(gpifile).write_text(gpitext, encoding='utf-8', newline='\n')
    subprocess.run([GNUPLOT, gpifile], check=True)


def gen_dir_item(name, filename):
  kw = dict(
    id = html.escape(name),
    name = html.escape(name),
    path = urlescape(name) + "/",
  )
  return DIR_ITEM_TPL.substitute(**kw)


def gen_item(dir, name, filename):
  mtime = datetime.utcfromtimestamp(os.stat(filename).st_mtime).isoformat(sep=' ')
  kw = dict(
    id = html.escape(name),
    name = html.escape(name),
    path = urlescape(name),
    style_attr = '',
    mtime_id = mtime.replace(' ', '').replace('-', '').replace(':', ''),
    mtime = mtime.split(' ')[0],
  )
  tpl = DEFAULT_ITEM_TPL
  ext = path.splitext(filename)[0]
  types = mimetypes.MimeTypes()
  types.add_type('audio/m4a', '.m4a')
  types.add_type('video/mp4', '.m4v')
  types.add_type('video/x-ms-wmv', '.wmv')
  types.add_type('image/vnd.adobe.photoshop', '.psd')
  types.add_type('font/ttf', '.ttf')
  types.add_type('font/woff', '.woff')
  types.add_type('font/woff2', '.woff2')
  types.add_type('font/otf', '.otf')
  types.add_type('image/svg+xml', '.svg')
  types.add_type('application/octet-stream', '.dmg')
  types.add_type('audio/midi', '.mid')
  types.add_type('application/ogg', '.ogg')
  types.add_type('audio/ogg', '.oga')
  typ = types.guess_type(filename)
  typ = typ[0] if typ[0] else '?'
  ext = path.splitext(filename)[1]
  thumbfile = ''
  thumbfile_gif = ''
  css_classes = []
  w = 0
  h = 0
  exts_unsupported_by_sox = ('.m4a',)

  if typ == 'application/pdf':
    thumbfile, thumbhash = thumb_filename(filename)
    thumbs.append((thumbfile, thumbhash))
    if not os.access(thumbfile, os.R_OK):
      pool.apply_async(gen_thumb_img, (filename, thumbfile))
  elif typ.startswith('audio/') and typ != 'audio/midi':
    css_classes.append('audio')
    duration = 0.0
    thumbfile, thumbhash = thumb_filename(filename, ext='.png')
    if os.access(thumbfile, os.R_OK):
      duration = -1.0
    elif os.access(GNUPLOT, os.R_OK) and ext not in exts_unsupported_by_sox:
      duration = audio_duration_secs(filename)
    if duration == 0.0:
      thumbfile = ''
    elif duration > 0.0:
      thumbs.append((thumbfile, thumbhash))
      if not os.access(thumbfile, os.R_OK):
        # gen_audio_thumb(filename, thumbfile, duration)
        pool.apply_async(gen_audio_thumb, (filename, thumbfile, duration))
  elif typ.startswith('video/'):
    css_classes.append('video')
    thumbfile_gif, thumbhash = thumb_filename(filename, ext='.gif')
    thumbfile = path.splitext(thumbfile_gif)[0] + '.png'
    thumbs.append((thumbfile, thumbhash))
    if not os.access(thumbfile_gif, os.R_OK) or not os.access(thumbfile, os.R_OK):
      # gen_video_thumb_gif(filename, thumbfile_gif, thumbfile)
      pool.apply_async(gen_video_thumb_gif, (filename, thumbfile_gif, thumbfile))
  elif typ.startswith('image/'):
    if path.splitext(filename)[1] == '.gif':
      thumbfile_gif = basename(filename)
    w, h = image_size(filename)
    if w > 0:
      thumbfile, thumbhash = thumb_filename(filename, max(w, h))
      thumbs.append((thumbfile, thumbhash))
      if not os.access(thumbfile, os.R_OK):
        pool.apply_async(gen_thumb_img, (filename, thumbfile, w, h))
    else:
      print(f"{typ}\t{name}\t(image_size failed)")
  elif typ == 'application/x-shockwave-flash':
    css_classes.append('flash')
  elif typ == '?' or typ.startswith('application/'):
    kw["style_attr"] = 'style="display:none"'
    css_classes.append('bin')
  elif typ.startswith('font/'):
    css_classes.append('font')
  elif typ == 'text/html':
    css_classes.append('html')
  elif typ.find('/') != -1:
    css_classes.append(typ[:typ.find('/')])
  else:
    print(f"{typ}\t{name}")

  if thumbfile != '':
    if max(w, h) < IMG_MAX_SIZE and max(w, h) > 0:
      css_classes.append('small')
    if thumbfile == filename:
      thumb_path = path.basename(filename)
    else:
      thumb_path = path.relpath(pjoin(ROOT_DIR, thumbfile), pjoin(ROOT_DIR, dir))
    if thumbfile_gif.startswith(THUMBS_DIR + '/'):
      thumbfile_gif = path.relpath(pjoin(ROOT_DIR, thumbfile_gif), pjoin(ROOT_DIR, dir))
    kw["thumb_path"] = urlescape(thumb_path)
    kw["thumb_gif"] = urlescape(thumbfile_gif)
    if thumbfile_gif != '':
      css_classes.append('animated')
      if typ == 'image/gif':
        css_classes.append('gif')
    tpl = IMG_ITEM_TPL

  kw["css_class"] = ' ' + ' '.join(css_classes) if css_classes else ''

  return tpl.substitute(**kw)


ROOT_TITLE = "stuff"


def gen_title(dir):
  url = path.relpath(ROOT_DIR, pjoin(ROOT_DIR, dir))
  v = []
  if dir == '':
    v.append(f'<a>{html.escape(ROOT_TITLE)}</a>')
    dir = 'stuff'
  else:
    url = path.relpath(ROOT_DIR, pjoin(ROOT_DIR, dir))
    v.append(f'<a href="{html.escape(url)}">{html.escape(ROOT_TITLE)}</a>')
    parentdir = '.'
    pathv = dir.split('/')
    for i in range(len(pathv)):
      s = pathv[i]
      parentdir += '/' + s
      url = path.relpath(pjoin(ROOT_DIR, parentdir), pjoin(ROOT_DIR, dir))
      if i == len(pathv) - 1:
        v.append(f'<a>{html.escape(s)}</a>')
      else:
        v.append(f'<a href="{html.escape(url)}">{html.escape(s)}</a>')
  return dir, '/'.join(v)


def gen_index(dir, subdirs, files):
  dir = dir[2:]  # strip leading "./"
  print(f"generating index for {dir if dir else '.'}/")
  items = []
  files.sort(key=str.casefold)
  # sorted(sorted(l1), key=str.casefold)
  for f in files:
    if f == SELF_SCRIPT_NAME or f[0] == '.' or f == 'index.html' or f.endswith(".template.html"):
      continue
    items.append(gen_item(dir, f, pjoin(dir, f)))
  for f in subdirs:
    if f[0] == '.' or f == "_web":
      continue
    items.append(gen_dir_item(f, pjoin(dir, f)))

  title, title_html = gen_title(dir)
  htmltext = INDEX_TPL.substitute(
    title = html.escape(title),
    title_html = title_html,
    items = '\n'.join(items),
    rooturl = urlescape(os.path.relpath(ROOT_DIR, ROOT_DIR + '/' + dir)),
  )
  indexfile = pjoin(dir, 'index.html')
  Path(indexfile).write_text(htmltext, encoding='utf-8', newline='\n')


def s3_sync(dry_run):
  # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/sync.html
  # https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters
  common_args = [
    'aws','s3','sync',
    '--no-follow-symlinks',
    '--delete',
    '--acl', 'public-read',
    '--exclude', '*.DS_Store',
  ]
  if dry_run:
    common_args.append('--dryrun')

  # sync thumbnails
  args = common_args + [
    '--expires', '2900-01-01T00:00:00Z',
    '--cache-control', 'max-age=31536000,immutable',
    './' + THUMBS_DIR,
    's3://d.rsms.me/stuff/' + THUMBS_DIR,
  ]
  env = dict(os.environ)
  env['AWS_CONFIG_FILE'] = '_web/aws-credentials.conf'
  print(f"s3 sync {THUMBS_DIR}/")
  subprocess.run(args, check=True, env=env)

  # sync everything else
  args = common_args + [
    '--exclude', THUMBS_DIR,
    '--exclude', MIRROR_DIR,
    '--exclude', '_web/gnuplot*',
    '--exclude', '_web/aws*',
    '--exclude', '_gen.py',
    '--exclude', '*.template.html',
    './',
    's3://d.rsms.me/stuff/',
  ]
  env = dict(os.environ)
  env['AWS_CONFIG_FILE'] = '_web/aws-credentials.conf'
  print("s3 sync ./")
  subprocess.run(args, check=True, env=env)


# ————————————————————————————————————————————————————————————————————————————————————

try:
  os.makedirs(THUMBS_DIR)
except FileExistsError:
  pass

for root, subdirs, files in os.walk("."):
  try: subdirs.remove('hunch-digest-blog-archive')
  except: pass
  try: subdirs.remove('_web')
  except: pass
  gen_index(root, subdirs, files)

# wait for background jobs
pool.close()
pool.join()

if len(sys.argv) < 2 or sys.argv[1] != '--no-s3':
  s3_sync(dry_run=False)

_index.template.html

<!DOCTYPE HTML>
<html lang="en">
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>$title</title>
  <style type="text/css">

body {
  background: #ddd;
  color: black;
  padding: 0;
  margin: 32px;
  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Noto Sans", Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji";
  font-size: 12px;
  font-weight: 500;
  line-height: 14px;
  --hover-bgcolor: #fff;
}

a {
  text-decoration: none;
}
label {
  user-select: none;
  -webkit-user-select: none;
  -webkit-touch-callout: none;
  cursor: default;
}

footer {
  display: block;
  color: #aaa;
  padding: 32px 16px;
}
footer a { color:inherit; text-decoration:underline; }
footer a:hover { color:black; }

.controls {
  padding: 16px;
  padding-top:0;
  display: flex;
}
.controls h1 {
  margin:0;
  flex: 2 1 auto;
  overflow: hidden;
  white-space: nowrap;
  text-overflow: ellipsis;
  font-size:16px;
}
.controls * { line-height: 32px; }
.controls h1 a { display:inline-block; color:inherit; margin:0 4px }
.controls h1 a:first-child { margin-left:0 }
.controls h1 a[href]:hover { text-decoration: underline }
.controls label { margin-left: 12px; }
.controls label.disabled { opacity: 0.3; }

.items {
  display: flex;
  flex-direction: column;
  align-items: start;
  margin: 0 auto;
}
a.item {
  flex: 0 1 auto;
  display: block;
}
a.item span.name {
  display: inline-block;
  cursor: inherit;
}
a.item.img .ani-overlay {
  display:none;
}

.items.list {
  padding: 0 12px 16px 12px;
  font-size: 16px;
  line-height: 22px;
  align-items: stretch;
}
.items.list a.item {
  background-size: 0;
  display: flex;
  flex: 1 0 auto;
  gap: 8px;
  padding-left: 4px;
}
.items.list .name {
  text-decoration: underline;
  flex: 1 1 auto;
  overflow: hidden;
  text-overflow: ellipsis;
  white-space: nowrap;
}
.items.list .mtime {
  text-align: right;
  white-space: nowrap;
}
.items.list a.item:hover {
  background-color: white;
}

.items.grid {
  flex-direction: row;
  flex-wrap: wrap;
  align-items: center;
  justify-content: start;
}

.items.grid a.item {
  display: flex;
  position: relative;
  overflow: hidden;
  align-items: center;
  justify-content: center;
  width: 128px;
  height: 128px;
  padding: 0px;
  margin: 16px;
  background-size: contain;
  background-repeat: no-repeat;
  background-position: center;
}
.items.grid a.item:hover {
  background-color: var(--hover-bgcolor);
  outline: 4px solid var(--hover-bgcolor);
}
.items.grid a.item span.name {
  display: block;
  margin: 24px;
  text-align: center;
  color: white;
}
.items.grid .mtime {
  display: none;
}

.items.grid a.item.dir {
  background-image: url("$rooturl/_web/dir.png");
  background-size: 100%;
}
.items.grid a.item.dir span.name {
  font-weight: 600;
  color: black;
  text-shadow: 0px 1px 1px white;
}

.items.grid a.item.default {
  background: #444;
  border-radius: 4px;
}
.items.grid a.item.default span.name {
  color: white;
  text-shadow: 0px 1px 1px black;
}
.items.grid a.item.default:hover {
  background: #555;
}

.items.grid a.item.type-mov { background-image:url(_web/mov.png) }

.items.grid a.item.img.small {
  image-rendering: pixelated;
}
.items.grid a.item.img span.name {
  pointer-events: none;
  font-weight: 600;
  display: flex;
  align-items: center;
  justify-content: center;
  position: absolute;
  top: 0; right: 0; bottom: 0; left: 0;
  color: transparent;
}
.items.grid a.item.img span.name::selection {
  color: black;
}
.items.grid a.item.img:hover span.name {
  color: white;
  text-shadow: 0px 0px 3px black;
}

.items.grid a.item.img .ani-overlay {
  display:none;
  position: absolute;
  top: 0; right: 0; bottom: 0; left: 0;
  background-size: contain;
  background-repeat: no-repeat;
  background-position: center;
  background-color: var(--hover-bgcolor);
  pointer-events:none;
}
.items.grid a.item.img.animated:hover .ani-overlay {
  display:block;
}

@media only screen and (max-width: 600px) {
  body { margin: 24px; }
  .controls {
    display: grid;
    grid-template-columns: repeat(3,1fr);
  }
  .items.list { padding:0; line-height:32px; }
  .controls h1 { grid-column: 1/-1; padding-bottom:16px; }
  .controls label { margin-left: 0; font-size:16px; line-height:24px; }
  .controls label.disabled { display: none; }
}

  </style>
</head>
<body>
<div class="controls">
  <h1>$title_html</h1>
  <label title="Display as list instead of grid"><input type=checkbox name=list-view> list</label>
  <label title="Sort by modification time"><input type=checkbox name=sort-mtime> mtime</label>
</div>
<div class="items grid">
$items
</div>
<footer>
  <a href="https://gist.github.com/rsms/9925894f0fff266d392bc149c648e535">Source code</a>
  &nbsp;•&nbsp;
  Have an issue with some content? Contact rasmus+stuff(AT)rsms·me
</footer>
<script type="text/javascript">(function(){

const Q = (sel, el) => [].slice.apply((el||document).querySelectorAll(sel))

const controls_container = Q('.controls')[0]
const items_container = Q('.items')[0]

// type flags
let iota = 0
const T_DIR   = 1 << iota++
const T_GIF   = 1 << iota++
const T_VIDEO = 1 << iota++
const T_AUDIO = 1 << iota++
const T_BIN   = 1 << iota++
const cssclass_to_flags = {
  'gif':   T_GIF,
  'video': T_VIDEO,
  'audio': T_AUDIO,
  'bin':   T_BIN,
  'dir':   T_DIR,
}

let filter = 0 // T_ flags
let all_filters = 0
let list_mode = (location.search == "?list")
let sort = 'name'

const items = Q('.item', items_container).map((el, index) => {
  const cl = el.classList
  let flags = 0
  for (let cls in cssclass_to_flags)
    flags |= cl.contains(cls) ? cssclass_to_flags[cls] : 0
  all_filters |= flags
  let default_display = el.style.display
  let name = el.innerText
  let mtime = el.dataset.mtime
  return { el, flags, default_order:index, name, mtime, default_display }
})

function set_filter(flags, on) {
  on ? filter |= flags : filter &= ~flags
}

function update_list_mode() {
  items_container.classList.toggle('grid', !list_mode)
  items_container.classList.toggle('list', list_mode)
}

function update_sort() {
  // function name_only_cmp(a,b) {
  //   return a.toLowerCase().localeCompare(b.toLowerCase())
  // }
  function name_only_cmp(a,b) {
    // use default sorting order in case there's a difference between
    // the web browser and the page generator
    return a.default_order - b.default_order
  }
  function name_cmp(a,b) {
    // always put dirs last
    if (!((a.flags & T_DIR) && (b.flags & T_DIR))) {
      if (a.flags & T_DIR)
        return 1
      if (b.flags & T_DIR)
        return -1
    }
    return name_only_cmp(a,b)
  }
  function mtime_cmp_r(a,b) {
    // always put dirs last
    if ((a.flags & T_DIR) && (b.flags & T_DIR))
      return name_only_cmp(a,b)
    if (a.flags & T_DIR)
      return 1
    if (b.flags & T_DIR)
      return -1
    return b.mtime < a.mtime ? -1 : a.mtime < b.mtime ? 1 : name_only_cmp(a,b)
  }
  items.sort(sort == 'mtime' ? mtime_cmp_r : name_cmp)
  items_container.style.display = 'none'
  items_container.innerHTML = ''
  for (let item of items)
    items_container.appendChild(item.el)
  items_container.style.display = null
}

function update_filter() {
  items_container.style.display = 'none'
  if (filter == 0) {
    for (let item of items)
      item.el.style.display = item.default_display
  } else {
    for (let item of items)
      item.el.style.display = (filter & item.flags) ? null : 'none'
  }
  items_container.style.display = null
}

for (let cls in cssclass_to_flags) {
  let flags = cssclass_to_flags[cls]
  let label = document.createElement("LABEL")
  label.innerHTML = `<input type=checkbox name="filter-$${cls}"> $${cls}`
  label.title = `Filter by type: $${cls}`
  controls_container.appendChild(label)
  if (flags & all_filters) {
    Q('[name=filter-'+cls+']')[0].onchange = ev => {
      set_filter(flags, ev.target.checked)
      update_filter()
    }
  } else {
    label.classList.add("disabled")
    label.children[0].disabled = true
  }
}

Q('[name=list-view]')[0].onchange = ev => {
  list_mode = ev.target.checked
  history.replaceState({}, null, document.location.pathname + (list_mode ? '?list' : ''))
  update_list_mode()
}
Q('[name=list-view]')[0].checked = list_mode

Q('[name=sort-mtime]')[0].onchange = ev => {
  sort = ev.target.checked ? 'mtime' : 'name'
  update_sort()
}

update_list_mode()

})()
</script>
</body>
</html>