Bulk downloader for Apple open source projects

aapl_slurp.rb

#!/usr/bin/env ruby

# aapl_slurp.rb: downloader for Apple open source projects
# Copyright © 2017, Ramsey Dow. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause

# mkdir /tmp/macOS-1013 && pushd !#:1
# curl -#LO https://opensource.apple.com/plist/macos-1013.plist
# SSL_CERT_FILE=$CURL_CA_BUNDLE ruby /path/to/aapl_slurp.rb macos-1013.plist

require 'open-uri'
require 'plist'

# https://github.com/patsplat/plist
# Note 'Security considerations'.

SCRIPT = File.basename($0, '.rb')

BASE = 'opensource.apple.com'
PATH = 'tarballs'

# Supah sleazy…
def parse_version(s)
  base = File.basename(s)
  first = base.split('.')
  second = first[0].split('-')
  major = second[1][0..1]
  minor = second[1][2..-1]
  return major,minor
end

def construct_url(name, version)
  "https://#{BASE}/#{PATH}/#{name}/#{name}-#{version}.tar.gz"
end

def download_file(url)
  File.open(File.basename(url), "wb") do |f|
    # Avoid loading entire response into memory…
    IO.copy_stream(open(url, "rb", :read_timeout => 60), f)
  end
end

if ARGV.size != 1
  puts "Usage: #{SCRIPT} <manifest.plist>"
  exit 1
end

major,minor = parse_version(ARGV[0])

result = Plist.parse_xml(ARGV[0])
if result.nil?
  puts "#{SCRIPT}: #{ARGV[0]}: error processing file"
  exit 1
end

build = result['build']
puts "macOS #{major}.#{minor}, build #{build}"

result['projects'].each do |project|
  name = project[0]
  version = project[1]['version']
  url = construct_url(name, version)
  begin
    download_file(url)
    puts "+ #{name} #{version}"
  rescue OpenURI::HTTPError
    puts "- #{name} #{version} → 404, unavailable"
    next
  rescue Net::OpenTimeout
    puts "#{name} #{version} fetch timed out"
    next
  end
end

exit 0