download all images from MKBHD's https://panels.art

openpanels.csx

//downloads all image data from https://storage.googleapis.com/panels-cdn/
//add auth to your buckets guys 🧐
using System.Text.Json;
using System.IO	;

var allUrl = "https://storage.googleapis.com/panels-cdn/data/20240730/all.json";
var json = await new System.Net.Http.HttpClient().GetStringAsync(allUrl);

List<string> urls = new();
var doc = JsonDocument.Parse(json);
ExtractUrls(doc.RootElement, urls);

Console.WriteLine($"found {urls.Count} urls");
if(!Directory.Exists("downloads"))
{
	Directory.CreateDirectory("downloads");
}

var options = new ParallelOptions { MaxDegreeOfParallelism = 10 };
Parallel.ForEach(urls, options, url =>
{
	var fileName = Path.GetFileName(url);
	var filePath = Path.Combine("downloads", fileName);
	if (!File.Exists(filePath))
	{
		Console.WriteLine($"downloading {url}");
		using var client = new System.Net.Http.HttpClient();
		var data = client.GetByteArrayAsync(url).Result;
		File.WriteAllBytes(filePath, data);
	}
	else
	{
		Console.WriteLine($"skipping {url}");
	}
});

static void ExtractUrls(JsonElement element, List<string> urls)
{
	if (element.ValueKind == JsonValueKind.Object)
	{
		foreach (var property in element.EnumerateObject())
		{
			if (property.Name == "url")
			{
				urls.Add(property.Value.GetString());
			}
			else
			{
				ExtractUrls(property.Value, urls);
			}
		}
	}
	else if (element.ValueKind == JsonValueKind.Array)
	{
		foreach (var item in element.EnumerateArray())
		{
			ExtractUrls(item, urls);
		}
	}
}