QMap Command Line

Program.cs

using HtmlAgilityPack;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;

namespace QMap.CLI
{
    public class Program
    {
        public static void Main(string[] args)
        {
            MainAsync(args).GetAwaiter().GetResult();
        }

        public static async Task MainAsync(string[] args)
        {
            Console.Title = "QMap CLI";
            //await Task.Run(() => Console.WriteLine("QMap Command Line Interface"));
            //await ParseKidsQMap();
            //await ParseEOsQMap();
            //await ParseIndictmentsQMap();
            //await ParseResignationsQMap();
            //await ParsePlayersQMap();
            await ParseThemesQMap();
        }

        public static async Task ParseKidsQMap()
        {
            // https://qmap.pub/kids?pg=1 - 13
            // Pages
            int[] range = Enumerable.Range(1, 1).ToArray();

            List<QMapKidsItem> list = new List<QMapKidsItem>();

            HttpClient httpClient = new HttpClient();

            foreach(int page in range)
            {
                var html = await httpClient.GetStringAsync(string.Format("https://qmap.pub/kids?pg={0}", page));

                var items = ParseKidsList(html);

                if (items is null)
                {
                    continue;
                }

                if (!items.Any())
                {
                    continue;
                }

                list.AddRange(items);
            }

            Console.WriteLine(JsonConvert.SerializeObject(list));
        }
        public static async Task ParseEOsQMap()
        {
            // https://qmap.pub/docs?pg=1 - 2
            // Pages
            int[] range = Enumerable.Range(1, 1).ToArray();

            List<QMapExecutiveOrderItem> list = new List<QMapExecutiveOrderItem>();

            HttpClient httpClient = new HttpClient();

            foreach (int page in range)
            {
                var html = await httpClient.GetStringAsync(string.Format("https://qmap.pub/docs?pg={0}", page));

                var items = ParseExecutiveOrdersList(html);

                if (items is null)
                {
                    continue;
                }

                if (!items.Any())
                {
                    continue;
                }

                list.AddRange(items);
            }

            Console.WriteLine(JsonConvert.SerializeObject(list));
        }
        public static async Task ParseIndictmentsQMap()
        {
            // https://qmap.pub/indictments?pg=1 - 14
            // Pages
            int[] range = Enumerable.Range(1, 1).ToArray();

            List<QMapIndictmentItem> list = new List<QMapIndictmentItem>();

            HttpClient httpClient = new HttpClient();

            foreach (int page in range)
            {
                var html = await httpClient.GetStringAsync(string.Format("https://qmap.pub/indictments?pg={0}", page));

                var items = ParseIndictmentsList(html);

                if (items is null)
                {
                    continue;
                }

                if (!items.Any())
                {
                    continue;
                }

                list.AddRange(items);
            }

            Console.WriteLine(JsonConvert.SerializeObject(list));
        }
        public static async Task ParseResignationsQMap()
        {
            // https://qmap.pub/indictments?pg=1 - 61
            // Pages
            int[] range = Enumerable.Range(1, 1).ToArray();

            List<QMapResignationItem> list = new List<QMapResignationItem>();

            HttpClient httpClient = new HttpClient();

            foreach (int page in range)
            {
                var html = await httpClient.GetStringAsync(string.Format("https://qmap.pub/resignations?pg={0}", page));

                var items = ParseResignationsList(html);

                if (items is null)
                {
                    continue;
                }

                if (!items.Any())
                {
                    continue;
                }

                list.AddRange(items);
            }

            Console.WriteLine(JsonConvert.SerializeObject(list));
        }

        public static async Task ParsePlayersQMap()
        {
            // https://qmap.pub/players?pg=1 - 5
            // Pages
            int[] range = Enumerable.Range(1, 1).ToArray();

            List<QMapPlayersItem> list = new List<QMapPlayersItem>();

            HttpClient httpClient = new HttpClient();

            foreach (int page in range)
            {
                var html = await httpClient.GetStringAsync(string.Format("https://qmap.pub/players?pg={0}", page));

                var items = ParsePlayersList(html);

                if (items is null)
                {
                    continue;
                }

                if (!items.Any())
                {
                    continue;
                }

                list.AddRange(items);
            }

            Console.WriteLine(JsonConvert.SerializeObject(list));
        }

        public static async Task ParseThemesQMap()
        {
            // https://qmap.pub/themes?pg=1 - 4
            // Pages
            int[] range = Enumerable.Range(1, 1).ToArray();

            List<QMapThemesItem> list = new List<QMapThemesItem>();

            HttpClient httpClient = new HttpClient();

            foreach (int page in range)
            {
                var html = await httpClient.GetStringAsync(string.Format("https://qmap.pub/themes?pg={0}", page));

                var items = ParseThemesList(html);

                if (items is null)
                {
                    continue;
                }

                if (!items.Any())
                {
                    continue;
                }

                list.AddRange(items);
            }

            Console.WriteLine(JsonConvert.SerializeObject(list));
        }

        public static List<QMapKidsItem> ParseKidsList(string html)
        {
            // XPath Notes
            //div[@id='posts']/div[1]
            //div[@id='posts']/div[1]/div[2]/a/@href = Article Url
            //div[@id='posts']/div[1]/div[1] = Date
            //div[@id='posts']/div[1]/div[2] = Name
            //div[@id='posts']/div[1]/div[3] = Desc
            //div[@id='posts']/div[1]/div[4] = Country Code
            //div[@id='posts']/div[1]/div[5] = State
            //div[@id='posts']/div[1]/div[6] = Arrested
            //div[@id='posts']/div[1]/div[7] = Rescued

            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

            doc.LoadHtml(html);

            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(".//div[@id='posts']/div");

            if (nodes == null)
            {
                return null;
            }

            Console.WriteLine("Total rows found: {0}", nodes.Count);

            List<QMapKidsItem> items = new List<QMapKidsItem>();

            foreach (HtmlNode node in nodes)
            {

                var newItem = new QMapKidsItem();

                try
                {
                    HtmlNode nameNode = node.SelectSingleNode(".//div[2]");
                    newItem.Name = nameNode.InnerText;
                    newItem.ReferenceUrl = nameNode.SelectSingleNode(".//a").GetAttributeValue("href", "");
                }
                catch (Exception)
                {
                    // No Name or URL Found
                }

                try
                {
                    HtmlNode descNode = node.SelectSingleNode(".//div[3]");
                    newItem.Description = descNode.InnerText;
                }
                catch (Exception)
                {
                    // No Desc Found
                }

                try
                {
                    HtmlNode countryCodeNode = node.SelectSingleNode(".//div[4]");
                    newItem.CountryCode = countryCodeNode.InnerText;
                }
                catch (Exception)
                {
                    // No Country Code Found
                }

                try
                {
                    HtmlNode stateNode = node.SelectSingleNode(".//div[5]");
                    newItem.State = stateNode.InnerText;
                }
                catch (Exception)
                {
                    // No State Found
                }

                try
                {
                    HtmlNode arrestedNode = node.SelectSingleNode(".//div[6]");
                    newItem.Arrested = Int32.Parse(arrestedNode.InnerText);
                }
                catch (Exception)
                {
                    // No Arrested Found
                }

                try
                {
                    HtmlNode rescuedNode = node.SelectSingleNode(".//div[7]");
                    newItem.Rescued = Int32.Parse(rescuedNode.InnerText);
                }
                catch (Exception)
                {
                    // No Rescued Found
                }

                try
                {
                    HtmlNode dateNode = node.SelectSingleNode(".//div[1]");
                    newItem.Date = dateNode.InnerText;
                }
                catch (Exception)
                {
                    // No Date Found
                }

                items.Add(newItem);
            }

            items = items.Where(x => !string.IsNullOrWhiteSpace(x.Name)).ToList();

            return items;
        }

        public static List<QMapExecutiveOrderItem> ParseExecutiveOrdersList(string html)
        {

            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

            doc.LoadHtml(html);

            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(".//div[@id='posts']/div");

            if (nodes == null)
            {
                return null;
            }

            Console.WriteLine("Total rows found: {0}", nodes.Count);

            List<QMapExecutiveOrderItem> items = new List<QMapExecutiveOrderItem>();

            foreach (HtmlNode node in nodes)
            {

                var newItem = new QMapExecutiveOrderItem();

                try
                {
                    HtmlNode titleNode = node.SelectSingleNode(".//div[4]");
                    newItem.Title = titleNode.InnerText;
                    
                }
                catch (Exception)
                {
                    // No Name or URL Found
                }

                try
                {
                    HtmlNode idNode = node.SelectSingleNode(".//div[1]");
                    newItem.EOId = idNode.InnerText;
                    newItem.ReferenceUrl = idNode.SelectSingleNode(".//a").GetAttributeValue("href", "");
                }
                catch (Exception)
                {
                    // No Desc Found
                }

                try
                {
                    HtmlNode dateNode = node.SelectSingleNode(".//div[2]");
                    newItem.SignedDate = dateNode.InnerText;
                }
                catch (Exception)
                {
                    // No Signed Date Found
                }

                try
                {
                    HtmlNode date2Node = node.SelectSingleNode(".//div[3]");
                    newItem.PublishedDate = date2Node.InnerText;
                }
                catch (Exception)
                {
                    // No Published Date Found
                }

                items.Add(newItem);
            }

            items = items.Where(x => !string.IsNullOrWhiteSpace(x.Title)).ToList();

            return items;
        }

        public static List<QMapIndictmentItem> ParseIndictmentsList(string html)
        {

            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

            doc.LoadHtml(html);

            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(".//div[@id='posts']/div");

            if (nodes == null)
            {
                return null;
            }

            Console.WriteLine("Total rows found: {0}", nodes.Count);

            List<QMapIndictmentItem> items = new List<QMapIndictmentItem>();

            foreach (HtmlNode node in nodes)
            {

                var newItem = new QMapIndictmentItem();

                try
                {
                    HtmlNode stateNode = node.SelectSingleNode(".//div[1]");
                    newItem.State = stateNode.InnerText;

                }
                catch (Exception)
                {
                    // No State Found
                }

                try
                {
                    HtmlNode dcNode = node.SelectSingleNode(".//div[2]");
                    newItem.DistrictCourt = dcNode.InnerText;

                }
                catch (Exception)
                {
                    // No District Court Found
                }

                try
                {
                    HtmlNode dateNode = node.SelectSingleNode(".//div[3]");
                    newItem.Date = dateNode.InnerText;
                }
                catch (Exception)
                {
                    // No Date Found
                }

                try
                {
                    HtmlNode sealedNode = node.SelectSingleNode(".//div[4]");
                    newItem.SealedCases = Int32.Parse(sealedNode.InnerText);
                }
                catch (Exception)
                {
                    // No Sealed Cases Found
                }

                try
                {
                    HtmlNode unsealedNode = node.SelectSingleNode(".//div[5]");
                    newItem.UnsealedCases = Int32.Parse(unsealedNode.InnerText);
                }
                catch (Exception)
                {
                    // No Unsealed Cases Found
                }

                try
                {
                    HtmlNode nonsealedNode = node.SelectSingleNode(".//div[6]");
                    newItem.NonSealedCases = Int32.Parse(nonsealedNode.InnerText);
                }
                catch (Exception)
                {
                    // No Non-Sealed Cases Found
                }

                items.Add(newItem);
            }

            items = items.Where(x => !string.IsNullOrWhiteSpace(x.State)).ToList();

            return items;
        }

        public static List<QMapResignationItem> ParseResignationsList(string html)
        {

            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

            doc.LoadHtml(html);

            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(".//div[@id='posts']/div");

            if (nodes == null)
            {
                return null;
            }

            Console.WriteLine("Total rows found: {0}", nodes.Count);

            List<QMapResignationItem> items = new List<QMapResignationItem>();

            foreach (HtmlNode node in nodes)
            {

                var newItem = new QMapResignationItem();

                try
                {
                    HtmlNode dateNode = node.SelectSingleNode(".//div[1]");
                    newItem.Date = dateNode.InnerText;
                }
                catch (Exception)
                {
                    // No Date Found
                }

                try
                {
                    HtmlNode nameNode = node.SelectSingleNode(".//div[2]");
                    newItem.Name = nameNode.InnerText;
                    newItem.ReferenceUrl = nameNode.SelectSingleNode(".//a").GetAttributeValue("href", "");
                }
                catch (Exception)
                {
                    // No Name Found
                }

                try
                {
                    HtmlNode roleNode = node.SelectSingleNode(".//div[3]");
                    newItem.Role = roleNode.InnerText;

                }
                catch (Exception)
                {
                    // No Role Found
                }

                try
                {
                    HtmlNode countryNode = node.SelectSingleNode(".//div[4]");
                    newItem.Country = countryNode.InnerText;
                }
                catch (Exception)
                {
                    // No Country Found
                }

                try
                {
                    HtmlNode orgNode = node.SelectSingleNode(".//div[5]");
                    newItem.Organization = orgNode.InnerText;
                }
                catch (Exception)
                {
                    // No Organization Found
                }

                try
                {
                    HtmlNode descNode = node.SelectSingleNode(".//div[6]");
                    newItem.Description = descNode.InnerText;
                }
                catch (Exception)
                {
                    // No Description Found
                }


                items.Add(newItem);
            }

            items = items.Where(x => !string.IsNullOrWhiteSpace(x.Name)).ToList();

            return items;
        }

        public static List<QMapPlayersItem> ParsePlayersList(string html)
        {

            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

            doc.LoadHtml(html);

            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(".//div/div[@class='card-columns']/div[@class='card']");

            if (nodes == null)
            {
                return null;
            }

            Console.WriteLine("Total rows found: {0}", nodes.Count);

            List<QMapPlayersItem> items = new List<QMapPlayersItem>();

            foreach (HtmlNode node in nodes)
            {

                var newItem = new QMapPlayersItem();

                try
                {
                    HtmlNode nameNode = node.SelectSingleNode(".//div[1]/h5");
                    newItem.Name = nameNode.InnerText;

                }
                catch (Exception)
                {
                    // No Title Found
                }

                try
                {                    
                    newItem.ImageUrl = node.SelectSingleNode(".//img").GetAttributeValue("data-src", "");

                }
                catch (Exception)
                {
                    // No Image URL Found
                }

                try
                {
                    HtmlNode descNode = node.SelectSingleNode(".//div[1]/p");
                    newItem.Description = descNode.InnerText;

                }
                catch (Exception)
                {
                    // No Desc Found
                }

                try
                {
                    HtmlNode tagNode = node.SelectSingleNode(".//div[1]/div[2]/h5[1]");
                    newItem.Tag = tagNode.InnerText;

                }
                catch (Exception)
                {
                    // No Tag Found
                }

                try
                {
                    HtmlNode alignmentNode = node.SelectSingleNode(".//div[1]/div[2]/h5[2]");
                    newItem.Alignment = alignmentNode.InnerText;

                }
                catch (Exception)
                {
                    // No Alignment Found
                }


                items.Add(newItem);
            }

            items = items.Where(x => !string.IsNullOrWhiteSpace(x.Name)).ToList();

            return items;
        }

        public static List<QMapThemesItem> ParseThemesList(string html)
        {

            HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

            doc.LoadHtml(html);

            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(".//div/div[@class='card-columns mb-3 mt-4']/div[@class='card']");

            if (nodes == null)
            {
                return null;
            }

            Console.WriteLine("Total rows found: {0}", nodes.Count);

            List<QMapThemesItem> items = new List<QMapThemesItem>();

            foreach (HtmlNode node in nodes)
            {

                var newItem = new QMapThemesItem();

                try
                {
                    HtmlNode titleNode = node.SelectSingleNode(".//div[1]/h5");
                    newItem.Title = titleNode.InnerText;

                }
                catch (Exception)
                {
                    // No Title Found
                }

                try
                {
                    newItem.ImageUrl = node.SelectSingleNode(".//img").GetAttributeValue("data-src", "");

                }
                catch (Exception)
                {
                    // No Image URL Found
                }

                try
                {
                    HtmlNode descNode = node.SelectSingleNode(".//p");
                    newItem.Description = descNode.InnerText;

                }
                catch (Exception)
                {
                    // No Desc Found
                }

                items.Add(newItem);
            }

            items = items.Where(x => !string.IsNullOrWhiteSpace(x.Title)).ToList();

            return items;
        }
    }

    public class QMapKidsItem
    {
        public string Name { get; set; }
        public string Date { get; set; }
        public string Description { get; set; }
        public string CountryCode { get; set; }
        public string State { get; set; }
        public int Arrested { get; set; }
        public int Rescued { get; set; }
        public string ReferenceUrl { get; set; }
    }

    public class QMapIndictmentItem
    {
        public string State { get; set; }
        public string DistrictCourt { get; set; }
        public string Date { get; set; }
        public int SealedCases { get; set; }
        public int UnsealedCases { get; set; }
        public int NonSealedCases { get; set; }
    }

    public class QMapResignationItem
    {
        public string Date { get; set; }
        public string Name { get; set; }        
        public string Role { get; set; }
        public string Country { get; set; }
        public string Organization { get; set; }
        public string Description { get; set; }
        public string ReferenceUrl { get; set; }        
    }

    public class QMapExecutiveOrderItem
    {
        public string EOId { get; set; }
        public string ReferenceUrl { get; set; }
        public string SignedDate { get; set; }
        public string PublishedDate { get; set; }
        public string Title { get; set; }
    }

    public class QMapPlayersItem
    {
        public string Name { get; set; }
        public string ImageUrl { get; set; }
        public string Description { get; set; }
        public string Tag { get; set; }
        public string Alignment { get; set; }
    }

    public class QMapThemesItem
    {
        public string Title { get; set; }
        public string ImageUrl { get; set; }
        public string Description { get; set; }
    }
}