This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
namespace Scraper { | |
class Program { | |
static void Main(string[] args) { | |
Console.WriteLine("Hello World!"); | |
} | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"version": "0.2.0", | |
"configurations": [ | |
{ | |
"name": ".NET Core Launch (console)", | |
"type": "coreclr", | |
"request": "launch", | |
"preLaunchTask": "build", | |
"program": "${workspaceFolder}/bin/Debug/netcoreapp2.1/CraigslistScraper.dll", | |
"args": [], |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static HtmlNode GetHtml(string url) { | |
WebPage webpage = _browser.NavigateToPage(new Uri(url)); | |
return webpage.Html; | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static List<string> GetMainPageLinks(string url) | |
{ | |
var homePageLinks = new List<string>(); | |
var html = GetHtml(url); | |
var links = html.CssSelect("a"); | |
foreach (var link in links) | |
{ | |
if (link.Attributes["href"].Value.Contains(".html")) | |
{ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static List<string> GetMainPageLinks(string url) | |
{ | |
var homePageLinks = new List<string>(); | |
var html = GetHtml(url); | |
var links = html.CssSelect("a"); | |
foreach (var link in links) | |
{ | |
if (link.Attributes["href"].Value.Contains(".html")) | |
{ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static void Main(string[] args) | |
{ | |
var mainPageLinks = GetMainPageLinks("https://newyork.craigslist.org/d/computer-gigs/search/cpg"); | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public class PageDetails { | |
public string title { get; set; } | |
public string description { get; set; } | |
public string url { get; set; } | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static List<PageDetails> GetPageDetails(List<string> urls) | |
{ | |
var lstPageDetails = new List<PageDetails>(); | |
foreach (var url in urls) | |
{ | |
var htmlNode = GetHtml(url); | |
var pageDetails = new PageDetails(); | |
pageDetails.title = htmlNode.OwnerDocument.DocumentNode | |
.SelectSingleNode("//html/head/title").InnerText; | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static void Main(string[] args) { | |
var mainPageLinks = GetMainPageLinks("https://newyork.craigslist.org/d/computer-gigs/search/cpg"); | |
var lstGigs = GetPageDetails(mainPageLinks); | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static void Main(string[] args) { | |
Console.WriteLine("Please enter a search term:") | |
var searchTerm = Console.ReadLine(); | |
var mainPageLinks = GetMainPageLinks("https://newyork.craigslist.org/d/computer-gigs/search/cpg"); | |
var lstGigs = GetPageDetails(mainPageLinks, searchTerm); | |
} | |
static List < PageDetails > GetPageDetails(List < string > urls, string searchTerm) { | |
var lstPageDetails = new List < PageDetails > (); | |
foreach(var url in urls) { |
OlderNewer