This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
let ip_addresses = []; | |
let port_numbers = []; | |
request("https://sslproxies.org/", function(error, response, html) { | |
if (!error && response.statusCode == 200) { | |
const $ = cheerio.load(html); | |
$("td:nth-child(1)").each(function(index, value) { | |
ip_addresses[index] = $(this).text(); | |
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const options = { | |
url: "https://www.forextradingbig.com/10-facts-you-must-know-on-online-forex-trading/", | |
method: "GET", | |
proxy: proxyGenerator() | |
}; | |
request(options, function(error, response, html) { | |
if (!error && response.statusCode == 200) { | |
const $ = cheerio.load(html); | |
let article_headings = $("h2").text(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from selenium import webdriver | |
from selenium.webdriver.common.keys import Keys | |
from selenium.webdriver.common.by import By | |
from selenium.webdriver.support.ui import WebDriverWait | |
from selenium.webdriver.support import expected_conditions as EC | |
# initialize webdriver | |
PATH = "C:\Program Files (x86)\chromedriver.exe" | |
driver = webdriver.Chrome(PATH) | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.Collections.Generic; | |
using HtmlAgilityPack; | |
using ScrapySharp.Extensions; | |
using ScrapySharp.Network; | |
using System.IO; | |
using System.Globalization; | |
using CsvHelper; | |
namespace ScrapySharp_scraper |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static void Main(string[] args) | |
{ | |
Console.WriteLine("Please enter the Keyword :"); | |
var Keyword = Console.ReadLine(); | |
var adLinks = GetAdLinks("https://losangeles.craigslist.org/search/bbb?"); | |
var lstAdDetails = GetAdDetails(adLinks, Keyword); | |
ExportAdsToCsv(lstAdDetails, Keyword); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static void ExportAdsToCsv(List<AdDetails> lstAdDetails, string Keyword){ | |
using(var writer = new StreamWriter($@"/Users/guest/Desktop/ScrapySharp_scraper/CSVs/{Keyword}_{DateTime.Now.ToFileTime()}.csv")) | |
using(var csv = new CsvWriter(writer, CultureInfo.InvariantCulture)){ | |
csv.WriteRecords(lstAdDetails); | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static void Main(string[] args) | |
{ | |
Console.WriteLine("Please enter the Keyword :"); | |
var Keyword = Console.ReadLine(); | |
var adLinks = GetAdLinks("https://losangeles.craigslist.org/search/bbb?"); | |
var lstAdDetails = GetAdDetails(adLinks, Keyword); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static List<AdDetails> GetAdDetails(List<string> urls, string Keyword){ | |
var lstAdDetails = new List<AdDetails>(); | |
foreach (var url in urls){ | |
var htmlNode = GetHtml(url); | |
var AdDetails = new AdDetails(); | |
AdDetails.AdTitle = htmlNode.OwnerDocument.DocumentNode.SelectSingleNode("//html/head/title").InnerText; | |
var description = htmlNode.OwnerDocument.DocumentNode.SelectSingleNode("//html/body/section/section/section/section").InnerText; | |
AdDetails.AdDescription = description.Replace("\n \n QR Code Link to This Post\n \n \n", ""); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static void Main(string[] args) | |
{ | |
var adLinks = GetAdLinks("https://losangeles.craigslist.org/search/bbb?"); | |
var lstAdDetails = GetAdDetails(adLinks); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
static List<AdDetails> GetAdDetails(List<string> urls){ | |
var lstAdDetails = new List<AdDetails>(); | |
foreach (var url in urls){ | |
var htmlNode = GetHtml(url); | |
var AdDetails = new AdDetails(); | |
AdDetails.AdTitle = htmlNode.OwnerDocument.DocumentNode.SelectSingleNode("//html/head/title").InnerText; | |
var description = htmlNode.OwnerDocument.DocumentNode.SelectSingleNode("//html/body/section/section/section/section").InnerText; | |
AdDetails.AdDescription = description.Replace("\n \n QR Code Link to This Post\n \n \n", ""); |
NewerOlder