Skip to content

Instantly share code, notes, and snippets.

@paigecook
Last active December 15, 2015 07:09
Show Gist options
  • Save paigecook/5221158 to your computer and use it in GitHub Desktop.
Save paigecook/5221158 to your computer and use it in GitHub Desktop.
Cookies based authentication for NCrawler, overrides the base functionality of WebDownloaderV2
How to add custom authentication using cookies based authorization from POST based login page.
1. Create cookies aware web client so that we can obtain the required cookies.
2. Create CustomWebDownloader that inherits from WebDownloaderV2 and overrides the cookie behavior.
3. Create custom NCrawlerModule that will implement the CustomWebDownloader.
4. Get the required login cookis for the session.
5. Register CustomDownloadModule passing in the authorized cookies. Note that the last item registered with Autofac will be the one used, so our CustomWebDownloader will now replace the default WebDownloaderV2.
6. Crawl the site, it will not use the CustomWebDownloader for all links crawled.
using System.Net;
namespace Crawler
{
public class CookiesAwareWebClient : WebClient
{
private CookieContainer outboundCookies = new CookieContainer();
private CookieCollection inboundCookies = new CookieCollection();
public CookieContainer OutboundCookies
{
get { return outboundCookies; }
}
public CookieCollection InboundCookies
{
get { return inboundCookies; }
}
public bool IgnoreRedirects { get; set; }
protected override WebRequest GetWebRequest(System.Uri address)
{
var request = base.GetWebRequest(address);
if (request is HttpWebRequest)
{
(request as HttpWebRequest).CookieContainer = outboundCookies;
(request as HttpWebRequest).AllowAutoRedirect = !IgnoreRedirects;
(request as HttpWebRequest).UserAgent =
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705;)";
(request as HttpWebRequest).ContentType = "application/x-www-form-urlencoded";
}
return request;
}
protected override WebResponse GetWebResponse(WebRequest request)
{
var response = base.GetWebResponse(request);
if (response is HttpWebResponse)
{
inboundCookies = (response as HttpWebResponse).Cookies ?? inboundCookies;
}
return response;
}
}
}
using System.Net;
using NCrawler;
using Autofac;
using NCrawler.Interfaces;
namespace Crawler
{
public class CustomDownloaderModule : NCrawlerModule
{
private readonly CookieContainer _cookieContainer;
public CustomDownloaderModule(CookieContainer cookieContainer)
{
_cookieContainer = cookieContainer;
}
protected override void Load(ContainerBuilder builder)
{
base.Load(builder);
builder.Register(c => new CustomWebDownloader(_cookieContainer))
.As<IWebDownloader>()
.SingleInstance()
.ExternallyOwned();
}
public static void Setup(CookieContainer cookieContainer)
{
Setup(cookieContainer);
}
}
}
using System.Net;
using NCrawler.Services;
namespace Crawler
{
public class CustomWebDownloader : WebDownloaderV2
{
private readonly CookieContainer _cookieContainer;
public CustomWebDownloader(CookieContainer cookieContainer)
{
_cookieContainer = cookieContainer;
}
protected override void SetDefaultRequestProperties(HttpWebRequest request)
{
base.SetDefaultRequestProperties(request);
request.CookieContainer = _cookieContainer;
}
}
}
using System;
using NCrawler;
using NCrawler.Services;
using Module = Autofac.Module;
using NCrawler.HtmlProcessor;
using NCrawler.Interfaces;
public static Main(string[] args)
{
var authorizedCookies = GetAuthorizationCookie(new Uri("http://mysecuresite.com/login.html"));
var modules = new Module[] { new CustomDownloaderModule(authorizedCookies)};
NCrawlerModule.Setup(modules);
using(Crawler c = new Crawler("http://mysecuresite.com/", new HtmlDocumentProcessor()))
{
c.Crawl();
}
}
private static CookieContainer GetAuthorizationCookie(Uri loginPage)
{
CookieContainer cookies;
//Put all required form post data here.
var postData = new NameValueCollection
{
{"userid", "user1"},
{"pwd", "password"},
};
using (var client = new CookiesAwareWebClient())
{
client.IgnoreRedirects = false;
//Load Page via get request to initialize cookies...
client.DownloadData(loginPage);
//Add cookies to the outbound request.
client.OutboundCookies.Add(client.InboundCookies);
client.UploadValues(loginPage, "POST", postData);
//Add latest cookies (includes the authorization to the cookie collection)
client.OutboundCookies.Add(client.InboundCookies);
cookies = client.OutboundCookies;
}
if (cookies == null || cookies.Count == 0)
{
Console.Writeline("Authorization Cookies are null or empty.");
}
else
{
Console.Writeline("Authorization Cookies obtained.");
}
return cookies;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment