scrape.csx

#r "SqlClientHelpers.dll" // download from here: https://github.com/Beej126/SqlClientHelpers/releases/download/1.0/SqlClientHelpers.dll
using System;
using System.Net;
using System.Data;
using static System.Console;

SqlClientHelpers.Proc.DefaultConnectionString = "xxx"; 
var proc = new SqlClientHelpers.Proc("SpreaderDataLookup_p");
var update = new SqlClientHelpers.Proc("dbo.SpreaderData_Table_u");
var client = new WebClient();
DataTable table;
DateTime start;
TimeSpan duration;
DateTime timer;
int counter;
string data;
string setting;

while (1 == 1)
{
  proc.DataSet?.Clear();
  table = proc.ExecuteDataSet().Table0;
  if (table.Rows.Count == 0) break;

  start = DateTime.UtcNow;
  timer = start;
  counter = 0;

  foreach (DataRow r in table.Rows)
  {
    counter++;
    data = "";
    setting = "";
    
    //here's the beef...
    //extract:
    data = client.DownloaString( $"url?id={r["SpreaderID"]}&spd={r["Speed"]}&den={r["Density"]}&mode=product&sprd={r["SpreadQty"]}"); //TBD
    //transform:
    setting = data.Split('|')[1];
    //load:
    r["Setting"] = setting;

    //report every ten seconds
    if (DateTime.UtcNow.Subtract(timer).TotalSeconds > 10)
    {
      duration = DateTime.UtcNow.Subtract(start);
      WriteLine($"{counter} URLs fired in {duration:mm}:{duration:ss} (min:sec).");
      timer = DateTime.UtcNow;
    }
  }

  duration = DateTime.UtcNow.Subtract(start);
  WriteLine($"{table.Rows.Count} total URLs fired in {duration:hh}:{duration:mm}:{duration:ss}.\nNow updating database...");

  start = DateTime.UtcNow;
  update["@SpreaderData"] = table; //here's all that is necessary for BULK UPLOAD of ADO.Net DataTable, too easy!
  update.ExecuteNonQuery();
  duration = DateTime.UtcNow.Subtract(start);
  WriteLine($"SQL update Took {duration:ss} seconds.");
}