Example script to download the latest version of all packages and their attendant artifacts (ignoring compatibility constraints)

i_have_too_much_disk_space.jl

empty!(Base.DEPOT_PATH)
push!(Base.DEPOT_PATH, joinpath(@__DIR__, "depot"))

using Pkg
Pkg.update()
registry_dir = joinpath(@__DIR__, "depot", "registries", "General")
regdict = Pkg.Types.read_registry(joinpath(registry_dir, "Registry.toml"))["packages"]

ctx = Pkg.Types.Context()
all_pkgspecs = Pkg.Types.PackageSpec[]
for (uuid, entry) in regdict
    # Install each package, just blithely grabbing the latest version, not bothering
    # to do resolution or anything.  First off, because it's not really possible to
    # install all Julia packages simultaneously, secondly, because we cool like dat
    versions = Pkg.Operations.load_versions(joinpath(registry_dir, entry["path"]))
    latest_version = maximum(keys(versions))
    latest_treehash = versions[latest_version]
    push!(all_pkgspecs, Pkg.Types.PackageSpec(
        name=entry["name"],
        uuid=Base.UUID(uuid),
        version=latest_version,
        tree_hash=latest_treehash
    ))
end

# Filter out packages we've already downloaded:
pkgs_to_install = Tuple{Pkg.Types.PackageSpec, String}[]
for pkg in all_pkgspecs
    path = Pkg.Operations.source_path(ctx, pkg)
    ispath(path) && continue
    push!(pkgs_to_install, (pkg, path))
end

widths = [textwidth(pkg.name) for (pkg, _) in pkgs_to_install]
max_name = length(widths) == 0 ? 0 : maximum(widths)

@info("Attempting to download $(length(pkgs_to_install)) packages using $(ctx.num_concurrent_downloads) parallel downloads...")
jobs = Channel(ctx.num_concurrent_downloads)
results = Channel(ctx.num_concurrent_downloads)
@async begin
    for pair in pkgs_to_install
        put!(jobs, pair)
    end
end

for i in 1:ctx.num_concurrent_downloads
    @async begin
        for (pkg, path) in jobs
            archive_urls = Pair{String,Bool}[
                "https://geo.pkg.julialang.org/package/$(pkg.uuid)/$(pkg.tree_hash)" => true,
            ]
            success = Pkg.Operations.install_archive(archive_urls, pkg.tree_hash, path)
            if success
                Pkg.Operations.set_readonly(path)
                push!(results, true)
            else
                push!(results, false)
            end
        end
    end
end

for idx in 1:length(pkgs_to_install)
    pkg = pkgs_to_install[idx][1]
    happy = take!(results)
    if happy
        Pkg.Operations.printpkgstyle(ctx, :Installed, string(rpad(pkg.name * " ", max_name + 2, "─"), " v$(pkg.version)"))
    else
        @warn("Unable to download $(pkg.name)")
    end
end

# Next, download all artifacts (for ALL PLATFORMS) from each package:
artifacts_tomls = String[]
packages_dir = joinpath(@__DIR__, "depot", "packages")
for pdir in readdir(packages_dir)
    for vdir in readdir(joinpath(packages_dir, pdir))
        # Only look at top-level Artifacts.toml files for now...
        artifacts_toml = joinpath(packages_dir, pdir, vdir, "Artifacts.toml")
        if isfile(artifacts_toml)
            push!(artifacts_tomls, artifacts_toml)
        end
    end
end

@info("Scanning through $(length(artifacts_tomls)) Artifacts.toml files")

for artifacts_toml in artifacts_tomls
    dict = Pkg.Artifacts.load_artifacts_toml(artifacts_toml)
    for artifact_name in keys(dict)
        entries = dict[artifact_name]
        if !isa(dict[artifact_name], Vector)
            entries = [entries]
        end

        for entry in entries
            # Skip non-downloadable artifacts
            if !haskey(entry, "download")
                continue
            end

            # Use the first download that works
            tree_hash = Base.SHA1(entry["git-tree-sha1"])
            if Pkg.Artifacts.artifact_exists(tree_hash)
                continue
            end

            download_success = false
            for dl_info in entry["download"]
                if Pkg.Artifacts.download_artifact(tree_hash, dl_info["url"], dl_info["sha256"])
                    download_success = true
                    break
                end
            end

            if !download_success
                @warn("Unable to download artifact $(artifact_name)")
            end
        end
    end
end