CookieAwareWebClient.cs

    /// <summary>
    /// WebClient with Cookies support, UserAgent
    /// </summary>
    public class CookieAwareWebClient : WebClient
    {
        public CookieContainer CookieContainer { get; } = new CookieContainer();

        public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";

        protected override WebRequest GetWebRequest(Uri address)
        {
            WebRequest request = base.GetWebRequest(address);
            if (request is HttpWebRequest httpRequest)
            {
                httpRequest.CookieContainer = CookieContainer;
                httpRequest.UserAgent = UserAgent;

                httpRequest.ProtocolVersion = HttpVersion.Version11;

                httpRequest.Timeout = 50000;
            }
            return request;
        }

        /// <summary>
        /// Add X-Requested-With=XMLHttpRequest header to all requests (until ClearXmlHttpRequestHeader called)
        /// </summary>
        public void SetXmlHttpRequestHeader()
        {
            Headers.Set("X-Requested-With", "XMLHttpRequest");
        }

        public void ClearXmlHttpRequestHeader()
        {
            Headers.Remove("X-Requested-With");
        }

        /// <summary>
        /// Add referer header (usually the URL of the last requested page). Referer header is removed after each request
        /// </summary>
        public void SetReferer(string url)
        {
            Headers.Set("Referer", url);
        }
    }