RawNetCommsTest

namespace Bung
{
  using System;
  using System.Collections.Specialized;
  using System.IO;
  using System.Linq;
  using System.Net;
  using System.Net.Cache;
  using System.Text;

  using CsQuery;

  using Xunit;

  public class RawNetCommsTest
  {
    private const string TestPath = @"C:\";
    private const string Url1 = "http://ent.kotui.org.nz/client/pn/search/results?qu=newbks-pn&te=&dt=list&st=PD";
    //private const string Url2 = "http://ent.kotui.org.nz/client/pn/search/results.displaypanel.displaycell:detailclick/0/ent:$002f$002fSD_ILS$002f2094$002fSD_ILS:2094331/0/0?qu=newbks-pn&d=ent%3A%2F%2FSD_ILS%2F2094%2FSD_ILS%3A2094331%7EILS%7E0%7E299&dt=list&st=PD";
    
    [Fact]
    public void TestSomething()
    {
      string url2;
      WebHeaderCollection headerCollection; 
      var cookieContainer = new CookieContainer();
      
      {
        var request = (HttpWebRequest)WebRequest.Create(Url1);
        request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
        request.AllowAutoRedirect = true;
        request.CookieContainer = cookieContainer;
        request.Proxy = WebRequest.GetSystemWebProxy();
        request.Proxy.Credentials = CredentialCache.DefaultNetworkCredentials;
        request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0";
        
        using (var response = (HttpWebResponse)request.GetResponse())
        {
          /* save headers */
          headerCollection = response.Headers;

          /* save cookies */
          foreach (Cookie cookie in response.Cookies)
          {
            cookieContainer.Add(cookie);
          }

          var htmlGet = string.Empty;
          var s = response.GetResponseStream();
          if (s != null)
          {
            using (var responseReader = new StreamReader(s))
            {
              htmlGet = responseReader.ReadToEnd();
            }
          }

          // find an item to investigate with a POST
          url2 = request.RequestUri.GetLeftPart(UriPartial.Authority) + GetUrlToPost(htmlGet);          

          htmlGet.SaveAs(TestPath + "TestSomething_GET.html");
        }
      }

      if (!string.IsNullOrEmpty(url2))
      {
        var request2 = (HttpWebRequest)WebRequest.Create(url2);
        request2.Accept = "text/javascript, text/html, application/xml, text/xml, */*";
        request2.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
        request2.AllowAutoRedirect = true;
        request2.CachePolicy = new RequestCachePolicy(RequestCacheLevel.NoCacheNoStore);
        request2.CookieContainer = cookieContainer;

        // Some headers are special and can't be accessed via this method
        SetHeaders(headerCollection, request2);

        request2.Proxy = WebRequest.GetSystemWebProxy();
        request2.Proxy.Credentials = CredentialCache.DefaultNetworkCredentials;
        request2.Referer = Url1;
        request2.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0";

        // POST-specific things:
        var encoding = new ASCIIEncoding();
        var data = encoding.GetBytes("t:zoneid=twilightZone0");

        request2.Method = "POST";
        request2.ContentType = "application/x-www-form-urlencoded";
        request2.ContentLength = data.Length;
        request2.KeepAlive = true;

        var dataStream = request2.GetRequestStream();
        dataStream.Write(data, 0, data.Length);
        dataStream.Close();

        using (var response2 = (HttpWebResponse)request2.GetResponse())
        {
          var htmlPost = string.Empty;
          var s = response2.GetResponseStream();
          if (s != null)
          {
            using (var responseReader = new StreamReader(s))
            {
              htmlPost = responseReader.ReadToEnd();
            }
          }

          htmlPost.SaveAs(TestPath + "TestSomething_POST.html");
        }
      }
      else
      {
        string.Empty.SaveAs(TestPath + "TestSomething_POST.html");
      }
    }

    private static void SetHeaders(NameValueCollection source, HttpWebRequest destinationRequest)
    {
      var restrictedHeaders = new[]
        {
          "Accept",
          "Connection",
          "Content-Length",
          "Content-Type",
          "Date",
          "Expect",
          "Host",
          "If-Modified-Since",
          "Range",
          "Referer",
          "Transfer-Encoding",
          "User-Agent",
          "Proxy-Connection"
        };
     
      foreach (var key in source.AllKeys.ToList().Where(k => restrictedHeaders.All(rh => !String.Equals(rh, k, StringComparison.CurrentCultureIgnoreCase))))
      {
        destinationRequest.Headers.Add(key, source[key]);
      }
    }

    private static string GetUrlToPost(string html)
    {
      var dom = CQ.Create(html); 
      var fragment = CQ.CreateFragment(dom["div.results_cell"].FirstElement().InnerHTML);
      var onclick = fragment["div.displayDetailLink a"].Attr("onclick");

      var temp = onclick.Remove(0, "stopPropagation(event); updateZone('detailClick','".Length);
      return temp.Split('\'')[0];
    }
  }

  public static class SerializationHelper
  {
    public static void SaveAs(this string data, string path)
    {
      using (var writer = new StreamWriter(path))
      {
        writer.Write(data);
        writer.Flush();
      }
    }
  }
}