Skip to content

Instantly share code, notes, and snippets.

@Miha22
Created May 23, 2021 18:43
Show Gist options
  • Save Miha22/ec38e6e4014b72c3ece065a5ecff32cb to your computer and use it in GitHub Desktop.
Save Miha22/ec38e6e4014b72c3ece065a5ecff32cb to your computer and use it in GitHub Desktop.
using System;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace DownloadZips
{
class Program
{
public static readonly string url = "https://api.bitcoincharts.com/v1/csv/inactive_exchanges/";
public static readonly string url_main = "https://api.bitcoincharts.com/v1/csv/";
public static string HrefRegex() => @"<a href\s*=\s*(?:[""'](?<1>[^""']*)[""']|(?<1>\S+))>(?<name>\S+\.\S+\.\S+)</a>";
public static async Task Main(string[] args)
{
await DownloadZipAsync(url_main, "BTC-History-2");
RenameFiles(new DriveInfo("D"), "BTC-History-2", ".gz", "");
Console.ReadLine();
}
private static async Task DownloadZipAsync(string url, string folder)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
using HttpWebResponse response = (HttpWebResponse)request.GetResponse();
using StreamReader reader = new StreamReader(response.GetResponseStream());
Regex regex = new Regex(HrefRegex(), RegexOptions.Multiline);
MatchCollection matches = regex.Matches(reader.ReadToEnd());
if (matches.Count > 0)
{
foreach (Match match in matches)
{
if (match.Success)
{
string nameDirty = match.Groups["name"].Value;
string req = url + nameDirty;
Console.WriteLine("Started: " + req);
DriveInfo drive = new DriveInfo("D");
await DownloadUnzipAsync(req, string.Format("{0}{1}{2}{3}{4}", drive.RootDirectory, '\\', "BTC-History", '\\', nameDirty[0..^3]));
Console.WriteLine("Finished downloading " + url);
}
}
}
else
Console.WriteLine("No match!");
Console.WriteLine("Finished downloading all URLs.");
}
private static void RenameFiles(DriveInfo drive, string folder, string endswith, string replace)
{
Console.WriteLine($"Started renaming files in \"{string.Format("{0}{1}{2}", drive.RootDirectory, '\\', folder)}\" that ends with \"{endswith}\", replace: \"{replace}\"");
new DirectoryInfo(string.Format("{0}{1}{2}", drive.RootDirectory, '\\', folder))
.GetFiles()
.Where(f => f.Name.EndsWith(endswith))
.ToList()
.ForEach(f => { File.Move(f.FullName, f.FullName.Replace(endswith, replace)); });
Console.WriteLine($"Finished renaming files in folder: {folder}");
}
private static async Task DownloadUnzipAsync(string url, string name)
{
using var client = new WebClient();
client.Headers.Add("accept", "*/*");
byte[] filedata = await client.DownloadDataTaskAsync(new Uri(url, UriKind.Absolute));
using MemoryStream ms = new MemoryStream(filedata);
using FileStream decompressedFileStream = File.Create(name, filedata.Length);
using GZipStream decompressionStream = new GZipStream(ms, CompressionMode.Decompress);
decompressionStream.CopyTo(decompressedFileStream);
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment