Pixiv FANBOX downloader

fanboxdl.rb

#!/usr/bin/env ruby

require "net/http"
require 'net/https'
require "uri"
require "json"
require "fileutils"
require "date"

def fetch(url, sessid)
  url = URI(url)
  headers = {
    "origin": "https://www.fanbox.cc",
    "cookie": "FANBOXSESSID=#{sessid}",
    "user-agent": "fanboxdl (twitter: @sukukyon, monora.me)",
  }

  http = Net::HTTP.new(url.host, url.port)
  http.use_ssl = true if url.scheme == "https"

  response = http.start do |http|
    path = "#{url.path}?#{url.query}"
    http.request_get(path, headers).body
  end
end

def fetch_json(url, sessid)
  fetch(url, sessid).then{ JSON.parse(_1)["body"] }
end

def fetch_file(url, sessid, dst)
  if File.exists?(dst)
    # puts "Skipped downloading #{url}, destination already exists: #{dst}"
    return
  end
  puts "Downloading #{url} -> #{dst}"
  file = fetch(url, sessid)
  File.write(dst, file)
end

def fetch_images(sessid, dir, images, title, article_id)
  return if images.empty?

  puts "Fetching #{images.length} images for #{title} (ID: #{article_id})"
  FileUtils.mkdir_p(dir)
  images.each do |idx, image|
    dst = "#{dir}/#{idx}_#{image["id"]}.#{image["extension"]}"
    fetch_file(image["originalUrl"], sessid, dst)
  end
end

def fetch_files(sessid, dir, files, title, article_id)
  return if files.empty?

  puts "Fetching #{files.length} files for #{title} (ID: #{article_id})"
  FileUtils.mkdir_p(dir)
  files.each do |idx, f|
    dst = "#{dir}/#{idx}_#{f["id"]}_#{f["name"]}.#{f["extension"]}"
    fetch_file(f["url"], sessid, dst)
  end
end

def download(creator_id, sessid)
  url = "https://api.fanbox.cc/post.listCreator?creatorId=#{creator_id}&limit=10"
  has_any_update = true

  FileUtils.mkdir_p(creator_id)

  while url && has_any_update
    # sleep 0.5
    has_any_update = false

    json = fetch_json(url, sessid)
    json["items"].each do |item|
      id = item["id"]
      title = item["title"]


      item_json_encoded = item.to_json
      json_path = "#{creator_id}/#{id}.json"
      if File.exists?(json_path)
        if File.size(json_path) != item_json_encoded.bytesize
          json_path_to_rename = json_path + ".#{Date.today.strftime("%Y%m%d")}"
          puts "Article size differ from existing file. (#{json_path}: #{File.size(json_path)}, fetched: #{item_json_encoded.bytesize})"
          FileUtils.move(json_path, json_path_to_rename)
          puts "Renamed old file to #{json_path_to_rename}"

          redo
        end
      elsif item["body"] != nil
        puts "Saving article to #{creator_id}/#{id}.json"
        File.write(json_path, item.to_json)
        has_any_update = true
      end

      article_dir = "#{creator_id}/#{id}_#{title.gsub("/", "-")}"

      if item["body"]
        images = item["body"]["images"]
        files = item["body"]["files"]
        if images || files
          if images
            imagesWithIndex = images.map.with_index{|x, i| [i + 1, x] }.to_h
            fetch_images(sessid, article_dir, imagesWithIndex, title, id)
          end
          imagesCount = (images || []).length
          if files
            filesWithIndex = files.map.with_index{|x, i| [i + 1 + imagesCount, x] }.to_h
            fetch_files(sessid, article_dir, filesWithIndex, title, id)
          end
        end

        if blocks = item["body"]["blocks"]
          imageMap = item["body"]["imageMap"]
          fileMap = item["body"]["fileMap"]

          ids = blocks.map do
            case _1["type"]
            when "image" then _1["imageId"]
            when "file" then _1["fileId"]
            else nil
            end
          end.compact

          if imageMap
            imagesWithIndex = imageMap.values.map{ [ids.index(_1["id"]) + 1, _1] }.to_h
            fetch_images(sessid, article_dir, imagesWithIndex, title, id)
          end

          if fileMap
            filesWithIndex = fileMap.values.map{ [ids.index(_1["id"]) + 1, _1] }.to_h
            fetch_files(sessid, article_dir, filesWithIndex, title, id)
          end
        end
      end
    end

    url = json["nextUrl"]
  end
end

SESSID_ENV_KEY = "FANBOXSESSID"

if ARGV.size == 0
  puts "#{$0} [--all|--list] [fanbox id]"
  unless ENV.has_key?(SESSID_ENV_KEY)
    puts "Warning: #{SESSID_ENV_KEY} environment value must be set"
  end
  exit
end

unless ENV.has_key?(SESSID_ENV_KEY)
  puts "Error: #{SESSID_ENV_KEY} environment value must be set"
  exit 1
end

sessid = ENV.fetch(SESSID_ENV_KEY)

case ARGV.first
when "--all"
  json = fetch_json("https://api.fanbox.cc/creator.listFollowing", sessid)
  json.each{ download(_1["creatorId"], sessid) }
when "--list"
  json = fetch_json("https://api.fanbox.cc/creator.listFollowing", sessid)
  puts json.map{ "@#{_1["creatorId"]}" }
else
  ARGV.each do
    creator_id = _1.delete_prefix("@")
    download(creator_id, sessid)
  end
end