SharePoint Online Login using Httpwebrequest via .net 3.5

SPOLogin.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.Xml.Linq;
using System.IO;
using System.Security;
using System.Xml;

namespace IconNewsTaskScheduler
{
    class SPOLogin
    {
        string UserName;
        string Password;
        string SPUrl;
        string SiteUrl;

        AuthState CurrentAuthState { get; set; }
        private CookieContainer CookieStore { get; set; }
        string CookieheaderForNonAdfs = string.Empty;
        private string binarySecurityToken;

        private string dateFormat = "yyyy-MM-ddTHH:mm:ss.fff0000Z";

        public delegate void LogHandler(string message);
        public event LogHandler Log;

        public SPOLogin(string userName, string password, string spUrl, string siteUrl)
        {
            this.UserName = userName;
            this.Password = password;
            this.SPUrl = spUrl;//Used for generation of AccessToken in account like kms-integrator@unitst.onmicrosoft.com
            this.SiteUrl = siteUrl;
        }

        public void Init()
        {
            CookieStore = new CookieContainer();
            try
            {
                string stsAuthUrl = GetStsAuthUrl();
                if (CurrentAuthState == AuthState.ADFS)
                {
                    string assertionXml = GetAssertionXml(stsAuthUrl);
                    if (!string.IsNullOrEmpty(assertionXml))
                    {
                        binarySecurityToken = GetBinarySecurityToken(assertionXml);
                        if (!string.IsNullOrEmpty(binarySecurityToken))
                        {
                            Cookie TokenCookie = GetADFSLoginCookie();
                            CookieStore.Add(TokenCookie);
                        }
                    }
                }
                else
                {
                    binarySecurityToken = GetBinarySecurityToken();
                    CookieCollection cookieColl = GetAccessTokenCookie();
                    CookieStore.Add(cookieColl);

                    CookieheaderForNonAdfs = cookieColl[0].ToStr() + ";" + cookieColl[1].ToStr();
                }
            }
            catch (Exception ex)
            {
                Log("SPoLogin.Init --> User Name and password may be incorrect" + ex.Message);
            }
        }

        private string GetStsAuthUrl()
        {
            string stsAuthUrl = string.Empty;
            string firstLoginUrl = "https://login.microsoftonline.com/getuserrealm.srf";
            string url = firstLoginUrl + "?login=" + UserName + "&xml=1";

            HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
            req.Method = "POST";
            req.Accept = "application/json; odata=verbose";
            req.ContentType = "application/soap+xml; charset=utf-8";
            req.ContentLength = 0;

            string responseStr = GetResponseBody(req);

            XDocument doc = XDocument.Parse(responseStr);
            string authStateStr = doc.Descendants().ElementAt(1).Value;
            if (authStateStr == "3")
            {
                //for account like rgupta@unicef.org
                CurrentAuthState = AuthState.ADFS;
                stsAuthUrl = doc.Descendants().ElementAt(9).Value;//Get the ("STSAuthURL");
                Log("StsAuthUrl -> " + stsAuthUrl);
            }
            else if (authStateStr == "4")
            {
                CurrentAuthState = AuthState.NonADFS;
                //for account like kms-integrator@unitst.onmicrosoft.com
            }
            return stsAuthUrl;
        }

        private string GetAssertionXml(string stsAuthUrl)
        {
            try
            {
                string createdDate = DateTime.Now.ToUniversalTime().ToString(dateFormat);
                string expiresDate = DateTime.Now.AddMinutes(10).ToUniversalTime().ToString(dateFormat);
                string guidStr = Guid.NewGuid().ToString("D");

                string postXml = SPOPostXml.StsAuthXml;
                postXml = string.Format(postXml, UserName, Password, createdDate, expiresDate, guidStr, stsAuthUrl);

                HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(stsAuthUrl);
                req.Method = "POST";
                //req.Accept = "application/json; odata=verbose";
                req.ContentType = "application/soap+xml; charset=utf-8";

                byte[] byteArray = Encoding.ASCII.GetBytes(postXml);
                req.ContentLength = byteArray.Length;

                Stream dataStream = req.GetRequestStream();
                dataStream.Write(byteArray, 0, byteArray.Length);
                dataStream.Close();

                string response = GetResponseBody(req);

                XDocument xmldoc = XDocument.Parse(response);
                string assertionFullXml = xmldoc.Descendants().Where(xg => xg.Name.LocalName == "Assertion").First().ToString(SaveOptions.DisableFormatting);

                Log("AssertionXml Generated");
                return assertionFullXml;
            }
            catch (Exception e)
            {
                Log("Error in Assertion Xml: " + e.Message + Environment.NewLine + e.StackTrace);
            }
            return string.Empty;
        }

        private string GetBinarySecurityToken(string assertionXml="")
        {
            Log("Binary Token Generation");
            string url, postData;
            if (CurrentAuthState == AuthState.ADFS)
            {
                url = "https://login.microsoftonline.com/RST2.srf";
                postData = string.Format(SPOPostXml.RST2Xml, assertionXml, SiteUrl);
            }
            else
            {
                url = "https://login.microsoftonline.com/extSTS.srf";
                postData = string.Format(SPOPostXml.SecurityTokenXml, UserName, Password, SiteUrl);
            }


            HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
            req.Method = "POST";
            req.Accept = "application/json; odata=verbose";
            req.ContentType = "application/soap+xml; charset=utf-8";

            byte[] byteArray = Encoding.ASCII.GetBytes(postData);
            req.ContentLength = byteArray.Length;

            Stream dataStream = req.GetRequestStream();
            dataStream.Write(byteArray, 0, byteArray.Length);
            dataStream.Close();

            string responseStr = GetResponseBody(req);
            //Log(responseStr);

            XElement x = XElement.Parse(responseStr);
            string binarySecurityToken = x.Descendants().Where(xg => xg.Name.LocalName == "BinarySecurityToken").First().Value;

            Log("Binary Security Token Generated");

            return binarySecurityToken;
        }

        private Cookie GetADFSLoginCookie()
        {
            string tokenUrl = "/_vti_bin/idcrl.svc/";
            string authCookieUrl = SiteUrl + tokenUrl;
            Cookie cookie = null;

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(authCookieUrl);
            request.CookieContainer = new CookieContainer();
            request.Method = "GET";
            request.Headers.Add(HttpRequestHeader.Authorization, "BPOSIDCRL " + binarySecurityToken);
            request.Headers.Add("X-IDCRL_ACCEPTED", "t");

            HttpWebResponse webRes = (HttpWebResponse)request.GetResponse();

            if (webRes.Cookies.Count > 0)
            {
                cookie = webRes.Cookies[0];
                Log("Cookie Generated");
            }
            else
            {
                Log("Cookie Not Generated");
            }

            return cookie;
        }

        private CookieCollection GetAccessTokenCookie()
        {
            string accesssToken = "/_forms/default.aspx?wa=wsignin1.0";
            string url2 = SPUrl + accesssToken;

            HttpWebRequest req2 = (HttpWebRequest)HttpWebRequest.Create(url2);
            req2.Method = "POST";
            req2.ContentType = "application/x-www-form-urlencoded";
            req2.CookieContainer = new CookieContainer();

            byte[] byteArray2 = Encoding.ASCII.GetBytes(binarySecurityToken);
            req2.ContentLength = byteArray2.Length;

            Stream dataStream2 = req2.GetRequestStream();
            dataStream2.Write(byteArray2, 0, byteArray2.Length);
            dataStream2.Close();

            HttpWebResponse res2 = (HttpWebResponse)req2.GetResponse();
            CookieCollection accessTokenCookies = res2.Cookies;
            Log("Cookie Generated");

            return accessTokenCookies;
        }

        public string GetFormDigest()
        {
            string formDigest = string.Empty;
            try
            {
                var url = SiteUrl + "/_api/contextinfo";
                var request = (HttpWebRequest)WebRequest.Create(url);
                request.Method = "POST";
                request.ContentLength = 0;
                if (CurrentAuthState == AuthState.ADFS)
                {
                    request.CookieContainer = CookieStore;
                }
                else
                {
                    request.Headers.Add("Cookie", CookieheaderForNonAdfs);
                }

                string responseStr = GetResponseBody(request);
                var resultXml = XDocument.Parse(responseStr);
                var e = from j in resultXml.Descendants()
                        where j.Name == XName.Get("FormDigestValue", "http://schemas.microsoft.com/ado/2007/08/dataservices")
                        select j;
                formDigest = e.First().Value;

                Log("Form Request Digest Created");

            }
            catch (Exception ex)
            {
                Log("Form Digest Not Created " + ex.Message + ex.StackTrace);
            }
            return formDigest;
        }

        private string GetResponseBody(HttpWebRequest req)
        {
            string url = req.RequestUri.AbsoluteUri;
            Log("Called "+ req.Method +" ->" + url);
            string responseStr = string.Empty;
            try
            {
                HttpWebResponse res = (HttpWebResponse)req.GetResponse();
                if (res.StatusCode == HttpStatusCode.OK)
                {
                    Stream responseStream = res.GetResponseStream();
                    responseStr = new StreamReader(responseStream).ReadToEnd();
                    responseStream.Close();
                }
                res.Close();
            }
            catch (WebException e)
            {
                string pageContent = new StreamReader(e.Response.GetResponseStream()).ReadToEnd().ToString();
                Log(pageContent + e.StackTrace);
            }
            catch (Exception e)
            {
                Log(url);
                Log(e.Message + e.StackTrace);
            }
            Log("Call Complete ->" + url);
            return responseStr;
        }

        public void UpdateItem(string relativeUrl, string data)
        {
            string digest = GetFormDigest();
            byte[] byteArray = Encoding.ASCII.GetBytes(data);

            var url = SiteUrl + relativeUrl;
            var req = (HttpWebRequest)WebRequest.Create(url);

            req.Method = "POST";
            req.Accept = "application/json;odata=verbose";
            req.ContentType = "application/json;odata=verbose";
            req.Headers.Add("X-RequestDigest", digest);
            req.Headers.Add("X-HTTP-Method", "MERGE");
            req.Headers.Add("IF-MATCH", "*");

            if (CurrentAuthState == AuthState.ADFS)
            {
                req.CookieContainer = CookieStore;
            }
            else
            {
                req.Headers.Add("Cookie", CookieheaderForNonAdfs);
            }

            req.ContentLength = byteArray.Length;

            Stream dataStream = req.GetRequestStream();
            dataStream.Write(byteArray, 0, byteArray.Length);
            dataStream.Close();

            string s = GetResponseBody(req);
            if (string.IsNullOrEmpty(s))
            {
                Log("Item Updated @ " + url);
            }
        }

        public void AddItem(string relativeUrl, string data)
        {
            string digest = GetFormDigest();

            var url = SiteUrl + relativeUrl;
            byte[] byteArray = Encoding.ASCII.GetBytes(data);

            var req = (HttpWebRequest)WebRequest.Create(url);
            req.Method = "POST";
            req.Accept = "application/json;odata=verbose";
            req.ContentType = "application/json;odata=verbose";
            req.Headers.Add("X-RequestDigest", digest);
            req.ContentLength = byteArray.Length;

            if (CurrentAuthState == AuthState.ADFS)
            {
                req.CookieContainer = CookieStore;
            }
            else
            {
                req.Headers.Add("Cookie", CookieheaderForNonAdfs);
            }

            Stream dataStream = req.GetRequestStream();
            dataStream.Write(byteArray, 0, byteArray.Length);
            dataStream.Close();

            string s = GetResponseBody(req);
            if (string.IsNullOrEmpty(s))
            {
                Log("Item Added @ " + url);
            }
        }

        public void DeleteItem(string relativeUrl)
        {
            string digest = GetFormDigest();

            var url = SiteUrl + relativeUrl;
            var req = (HttpWebRequest)WebRequest.Create(url);

            req.Method = "POST";
            req.Accept = "application/json;odata=verbose";
            req.ContentType = "application/json;odata=verbose";
            req.Headers.Add("X-RequestDigest", digest);
            req.Headers.Add("X-HTTP-Method", "DELETE");
            req.Headers.Add("IF-MATCH", "*");
            req.ContentLength = 0;

            if (CurrentAuthState == AuthState.ADFS)
            {
                req.CookieContainer = CookieStore;
            }
            else
            {
                req.Headers.Add("Cookie", CookieheaderForNonAdfs);
            }

            string s = GetResponseBody(req);
            if (string.IsNullOrEmpty(s))
            {
                Log("Item Deleted @ " + url);
            }
        }

        public string AddFile(string relativeUrl, byte[] fileBytes)
        {
            string newPicUrl = string.Empty;
            try
            {
                string formDigest = GetFormDigest();

                HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(relativeUrl);
                req.Method = "POST";
                req.Accept = "application/xml";
                req.ContentType = "application/json;odata=verbose";
                req.Headers.Add("X-RequestDigest", formDigest);

                if (CurrentAuthState == AuthState.ADFS)
                {
                    req.Headers.Add("binaryStringRequestBody", "true");
                    req.CookieContainer = CookieStore;
                }
                else
                {
                    req.Headers.Add("Cookie", CookieheaderForNonAdfs);
                }

                req.ContentLength = fileBytes.Length;

                Stream dataStream = req.GetRequestStream();
                dataStream.Write(fileBytes, 0, fileBytes.Length);
                dataStream.Close();

                string responseStr = GetResponseBody(req);

                var resultXml = XDocument.Parse(responseStr);
                var e = from j in resultXml.Descendants()
                        where j.Name == XName.Get("ServerRelativeUrl", "http://schemas.microsoft.com/ado/2007/08/dataservices")
                        select j;
                newPicUrl = e.FirstOrDefault().Value;
            }
            catch (WebException e)
            {
                string pageContent = new StreamReader(e.Response.GetResponseStream()).ReadToEnd().ToString();
                Log("error in file upload: " + pageContent + e.StackTrace);
            }
            return newPicUrl;
        }

        public string CallRestApi(string url, string JsonOrXml = "json")
        {
            HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(SiteUrl + url);
            request.Method = "GET";
            request.Accept = "application/" + JsonOrXml;

            if (CurrentAuthState == AuthState.ADFS)
            {
                request.CookieContainer = CookieStore;
            }
            else
            {
                request.Headers.Add("Cookie", CookieheaderForNonAdfs);
            }

            string responseStr = GetResponseBody(request);
            return responseStr;
        }
    }

    enum AuthState
    {
        ADFS,
        NonADFS
    }
    
    class SPOPostXml
    {
        public static string StsAuthXml = @"<?xml version='1.0' encoding='UTF-8'?>
                                <s:Envelope xmlns:s='http://www.w3.org/2003/05/soap-envelope' xmlns:wsse='http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd' xmlns:saml='urn:oasis:names:tc:SAML:1.0:assertion' xmlns:wsp='http://schemas.xmlsoap.org/ws/2004/09/policy' xmlns:wsu='http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd' xmlns:wsa='http://www.w3.org/2005/08/addressing' xmlns:wssc='http://schemas.xmlsoap.org/ws/2005/02/sc' xmlns:wst='http://schemas.xmlsoap.org/ws/2005/02/trust'>
                                  <s:Header>
                                    <wsa:Action s:mustUnderstand='1'>http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue</wsa:Action>
                                    <wsa:To s:mustUnderstand='1'>{5}</wsa:To>
                                    <wsa:MessageID>{4}</wsa:MessageID>
                                    <ps:AuthInfo xmlns:ps='http://schemas.microsoft.com/Passport/SoapServices/PPCRL' Id='PPAuthInfo'>
                                      <ps:HostingApp>Managed IDCRL</ps:HostingApp>
                                      <ps:BinaryVersion>6</ps:BinaryVersion>
                                      <ps:UIVersion>1</ps:UIVersion>
                                      <ps:Cookies/>
                                      <ps:RequestParams>AQAAAAIAAABsYwQAAAAxMDMz</ps:RequestParams>
                                    </ps:AuthInfo>
                                    <wsse:Security>
                                      <wsse:UsernameToken wsu:Id='user'>
                                        <wsse:Username>{0}</wsse:Username>
                                        <wsse:Password>{1}</wsse:Password>
                                      </wsse:UsernameToken>
                                      <wsu:Timestamp Id='Timestamp'>
                                        <wsu:Created>{2}</wsu:Created>
                                        <wsu:Expires>{3}</wsu:Expires>
                                      </wsu:Timestamp>
                                    </wsse:Security>
                                  </s:Header>
                                  <s:Body>
                                    <wst:RequestSecurityToken Id='RST0'>
                                      <wst:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue</wst:RequestType>
                                      <wsp:AppliesTo>
                                        <wsa:EndpointReference>
                                          <wsa:Address>urn:federation:MicrosoftOnline</wsa:Address>
                                        </wsa:EndpointReference>
                                      </wsp:AppliesTo>
                                      <wst:KeyType>http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey</wst:KeyType>
                                    </wst:RequestSecurityToken>
                                  </s:Body>
                                </s:Envelope>";


        public static string RST2Xml = @"<S:Envelope xmlns:S='http://www.w3.org/2003/05/soap-envelope' xmlns:wsse='http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd' xmlns:wsp='http://schemas.xmlsoap.org/ws/2004/09/policy' xmlns:wsu='http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd' xmlns:wsa='http://www.w3.org/2005/08/addressing' xmlns:wst='http://schemas.xmlsoap.org/ws/2005/02/trust'>
                <S:Header>
	                <wsa:Action S:mustUnderstand='1'>http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue</wsa:Action>
	                <wsa:To S:mustUnderstand='1'>https://login.microsoftonline.com/rst2.srf</wsa:To>
	                <ps:AuthInfo xmlns:ps='http://schemas.microsoft.com/LiveID/SoapServices/v1' Id='PPAuthInfo'>
		                <ps:BinaryVersion>5</ps:BinaryVersion>
		                <ps:HostingApp>Managed IDCRL</ps:HostingApp>
	                </ps:AuthInfo>
	                <wsse:Security>
		                {0}
	                </wsse:Security>
                </S:Header>
                <S:Body>
	                <wst:RequestSecurityToken xmlns:wst='http://schemas.xmlsoap.org/ws/2005/02/trust' Id='RST0'>
		                <wst:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue</wst:RequestType>
		                <wsp:AppliesTo>
			                <wsa:EndpointReference>
				                <wsa:Address>{1}</wsa:Address>
			                </wsa:EndpointReference>
		                </wsp:AppliesTo>
		                <wsp:PolicyReference URI='MBI'/>
	                </wst:RequestSecurityToken>
                </S:Body>
                </S:Envelope>";
        //Replace urn:federation:MicrosoftOnline for https://xxx.sharepoint.com 

        public static string SecurityTokenXml = @"<s:Envelope xmlns:s='http://www.w3.org/2003/05/soap-envelope'
                      xmlns:a='http://www.w3.org/2005/08/addressing'
                      xmlns:u='http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd'>
                  <s:Header>
                    <a:Action s:mustUnderstand='1'>http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue</a:Action>
                    <a:ReplyTo>
                      <a:Address>http://www.w3.org/2005/08/addressing/anonymous</a:Address>
                    </a:ReplyTo>
                    <a:To s:mustUnderstand='1'>https://login.microsoftonline.com/extSTS.srf</a:To>
                    <o:Security s:mustUnderstand='1'
                       xmlns:o='http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd'>
                      <o:UsernameToken>
                        <o:Username>{0}</o:Username>
                        <o:Password>{1}</o:Password>
                      </o:UsernameToken>
                    </o:Security>
                  </s:Header>
                  <s:Body>
                    <t:RequestSecurityToken xmlns:t='http://schemas.xmlsoap.org/ws/2005/02/trust'>
                      <wsp:AppliesTo xmlns:wsp='http://schemas.xmlsoap.org/ws/2004/09/policy'>
                        <a:EndpointReference>
                          <a:Address>{2}</a:Address>
                        </a:EndpointReference>
                      </wsp:AppliesTo>
                      <t:KeyType>http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey</t:KeyType>
                      <t:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue</t:RequestType>
                      <t:TokenType>urn:oasis:names:tc:SAML:1.0:assertion</t:TokenType>
                    </t:RequestSecurityToken>
                  </s:Body>
                </s:Envelope>";
    }
    
    public static class ProjExtension
    {
        public static string ToStr(this object o)
        {
            if (o != null)
            {
                return o.ToString();
            }
            return string.Empty;
        }

        public static string ToLiteral(this string input)
        {
            var literal = new StringBuilder(input.Length + 2);
            //literal.Append("\"");
            foreach (var c in input)
            {
                switch (c)
                {
                    case '\'': literal.Append(@"\'"); break;
                    case '\"': literal.Append("\\\""); break;
                    case '\\': literal.Append(@"\\"); break;
                    case '\0': literal.Append(@"\0"); break;
                    case '\a': literal.Append(@"\a"); break;
                    case '\b': literal.Append(@"\b"); break;
                    case '\f': literal.Append(@"\f"); break;
                    case '\n': literal.Append(@"\n"); break;
                    case '\r': literal.Append(@"\r"); break;
                    case '\t': literal.Append(@"\t"); break;
                    case '\v': literal.Append(@"\v"); break;
                    default:
                        if (Char.GetUnicodeCategory(c) != UnicodeCategory.Control)
                        {
                            literal.Append(c);
                        }
                        else
                        {
                            literal.Append(@"\u");
                            literal.Append(((ushort)c).ToString("x4"));
                        }
                        break;
                }
            }
            //literal.Append("\"");
            return literal.ToString();
        }
    }
}

//Usage
SPOLogin login = new SPOLogin(cv.SPOUserName, pwd, "http://xxx.sharepoint.com/", "http://xxx.sharepoint.com/teams/xxx");
login.Log += new SPOLogin.LogHandler(Logger);
login.Init();

string responseItems = login.CallRestApi("/_api/web/lists/GetByTitle('IconNews')/items", "xml");

string onlinePicUrl = login.AddFile("/_api/web/Folders('IconNewsPicture')/Files/add(url='{0}',overwrite=true)", bytes[]);
login.UpdateItem("_api/web/lists/GetByTitle('IconNews')/items", data);
login.DeleteItem("_api/web/lists/GetByTitle('IconNews')/items(1)");
login.AddItem("_api/web/lists/GetByTitle('IconNews')/items", data);