moebooru-get.py

#!/usr/bin/env python3
from os.path import basename
import argparse
import json
import os
import sys
import urllib.parse
import urllib.request

if sys.stdout.isatty():
	def tprint(*args, **kwargs):
		if "flush" not in kwargs:
			kwargs['flush'] = True

		return print(*args, **kwargs)
else:
	tprint = lambda *a, **k: ...

def progress(cur, full):
  #chars = "▁▂▃▄▅▆▇█"
  chars = "⠁⠉⠋⠛⠟⠿⡿⣿"
  frac = cur / full
  index = 0

  for i in range(len(chars)):
    if abs(frac - (index+1)/len(chars)) >= abs(frac - (i+1)/len(chars)):
      index = i

  return chars[index]

def fetchall(baseurl, tags, incremental=False):
  pagenum = 1

  while True:
    url = "%s/post.json?tags=%s&page=%s" % (baseurl, tags, pagenum)
    cur = urllib.request.urlopen(url)
    cur = json.loads(cur.read().decode())
    if not cur: break

    for i in cur:
      print(i['id'], end=" ")
      sys.stdout.flush()
      filename = urllib.parse.unquote(i['file_url'].split('/')[-1])

      if os.path.exists(filename):
        print("✈")

        if incremental:
          return
        else:
          continue

      r = urllib.request.urlopen(i['file_url'])
      try:
        total = 0
        full = i['file_size']

        with open(filename, "wb") as f:
          while True:
            if total: tprint("\b", end='')
            tprint(progress(total, full), end='')

            chunk = r.read(1024)
            total += 1024

            if not chunk: break
            f.write(chunk)
            f.flush()
      except BaseException: # TODO add less
        tprint("\b", end='')
        print("✗")
        print("fug")
        print("removing incomplete", i['id'])
        os.remove(filename)
        return
      else:
        tprint("\b", end='')
        print("✓")

    pagenum += 1

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.add_argument("-i", "--incremental", action="store_true", help="stop fetching after encountering an already downloaded file")
  parser.add_argument("baseurl")
  parser.add_argument("tag", nargs="*")

  args = parser.parse_args()
  fetchall(args.baseurl, ' '.join(args.tag), args.incremental)

moebooru-poolget.py

#!/usr/bin/env python3
from os.path import basename
import argparse
import contextlib
import json
import os
import subprocess
import sys
import urllib.parse
import urllib.request

scriptdir = os.path.dirname(os.path.abspath(__file__))

def shellesc(s):
  return "'" + s.replace("'", r"'\''") + "'"

def poolget(baseurl, id):
  url = "%s/pool/show.json?id=%s" % (baseurl, id)
  pool = urllib.request.urlopen(url)
  pool = json.loads(pool.read().decode())
  total = len(pool['posts'])
  page = 1

  # we could save some requests on long pools by downloading the images directly here
  while total != pool['post_count']:
    page += 1
    print("%d/%d posts, dl page %d" % (total, pool['post_count'], page))
    moreposts = urllib.request.urlopen("%s&page=%s" % (url, page))
    moreposts = json.loads(moreposts.read().decode())

    pool['posts'].extend(moreposts['posts'])
    total += len(moreposts['posts'])

  with open("pool%s.json" % id, "w") as f:
    json.dump(pool, f)

  getscript = os.path.join(scriptdir, "moebooru-get.py")
  getcmd = [getscript, baseurl, "pool:%s" % id]

  print(' '.join(shellesc(x) for x in getcmd), file=sys.stderr)
  subprocess.run(getcmd)

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.add_argument("baseurl")
  parser.add_argument("id", nargs="*")

  args = parser.parse_args()
  for id in args.id:
    poolget(args.baseurl, id)