Planetside 2 Downloading Code

Notes.txt

Dependencies:
 * json.net: http://james.newtonking.com/json
 * LZMA: http://testoutfit.info/lebot/LZMA.dll (I had to custom build this.  It's not a virus trust me)


Manifest Locations:

Live: "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-live/live/planetside2-live.sha.soe"
Live-Last: "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-live/livelast/planetside2-live.sha.soe"

LiveCommon: "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-livecommon/live/planetside2-livecommon.sha.soe"
LiveCommon-Last: "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-livecommon/livelast/planetside2-livecommon.sha.soe"

Test: "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-test/live/planetside2-test.sha.soe"
Test-Last: "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-live/livelast/planetside2-live.sha.soe"

TestCommon: "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-testcommon/live/planetside2-testcommon.sha.soe"
TestCommon-Last: "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-testcommon/livelast/planetside2-testcommon.sha.soe"

Common manifests contain most of the files in the Planetside 2 install.  The Live or Test manifests only contain the exe, scripts, and uninstaller.

the -Last manifests are the Manifests for the previous version of the game.  These are used to determine if the file has been updated.  

This will not patch files, it will replace them.  The SOE launcher is more elegant here.  

Program.cs

static void Main(string[] args)
{
    string manifest = "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-live/live/planetside2-live.sha.soe";
    string lastmanifest = "http://manifest.patch.station.sony.com/patch/sha/manifest/planetside2/planetside2-live/livelast/planetside2-live.sha.soe";
      
    
    JObject jo =  PS2Downloader.GetJobjectFromManifest(manifest);
    
    
    JObject lo = PS2Downloader.GetJobjectFromManifest(lastmanifest);
    
    PS2Downloader.DownloadPS2Job(jo, null, "Live", j => { });
    
    while (PS2Downloader.DownloadJobsToComplete() > 0)
    {
        foreach (string status in PS2Downloader.ReportDownloadStatus())
        {
            Console.WriteLine(status);
        }
        Thread.Sleep(500);
    }
    Console.WriteLine("Done");
}

PS2Downloader.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.IO;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json;
using System.Collections.Concurrent;
using System.Threading;
using System.Threading.Tasks;



namespace REsideUtilities.PS2DownloadUtility
{
    class DownloadJob 
    {
        public string name;
        public string url;
        public string output;
        public bool decompress;
        public string status;
        public string jobtag;
     
    }

    delegate void FullDownloadJobComplete(string jobstart);

    static class PS2Downloader
    {
        static ConcurrentQueue<DownloadJob> DownloadQueue = new ConcurrentQueue<DownloadJob>();
        static List<DownloadJob> DownloadingQueue = new List<DownloadJob>();
        static ConcurrentDictionary<string, FullDownloadJobComplete> JobCallbacks = new ConcurrentDictionary<string, FullDownloadJobComplete>();

        static Thread DownloadWorker = null;

        static void EmptyJobCompleteCallback(string time)
        {
            return;
        }

        public static void DownloadPS2(JObject thisManifest, JObject lastManifest, string outputfolder)
        {
            DownloadPS2Job(thisManifest, lastManifest, outputfolder, EmptyJobCompleteCallback);
        }

        public static string DownloadPS2Job(JObject thisManifest, JObject lastManifest, string outputfolder, FullDownloadJobComplete jobComplete)
        {
            if (DownloadWorker == null)
            {
                DownloadWorker = new Thread(new ThreadStart(DownloadThreadWork));
                DownloadWorker.Start();
            }

            if (!Directory.Exists("output/")) Directory.CreateDirectory("output/");
            if (!Directory.Exists("output/PS2Install")) Directory.CreateDirectory("output/PS2Install");
            if (!Directory.Exists("output/PS2Install/" + outputfolder)) Directory.CreateDirectory("output/PS2Install/" + outputfolder);

            string of = "output/PS2Install/" + outputfolder +"/";


            JToken folders = thisManifest["digest"]["folder"];
            
            string shaasset = (string)thisManifest["digest"]["@shaAssetURL"];
          
            string jobStart = DateTime.Now.ToString();
            JobCallbacks[jobStart] = jobComplete;

            if (lastManifest == null) UpdateFilesInFolder(shaasset, of, folders, null, jobStart);
            else
            {
                JToken oldFolders = lastManifest["digest"]["folder"];
                UpdateFilesInFolder(shaasset, of, folders, oldFolders, jobStart);
            }

            return jobStart;

        }

        private static void UpdateFilesInFolder(string downloadurl, string folder, JToken thisFolder, JToken lastFolder, string job)
        {

            if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
            //Sometimes there is a null child
            if (thisFolder == null) return;
            //If the folder is an array of files (not a folder structure), go through each of the folders and parse them
            if (thisFolder.Type == JTokenType.Array)
            {
                foreach (var token in thisFolder)
                {
                    UpdateFilesInFolder(downloadurl, folder, token, lastFolder, job);
                }
                return;
            }


            JToken t = null;
            if (thisFolder["file"] != null)
            {
                if (thisFolder["file"].Type == JTokenType.Array)
                {
                    foreach (var file in thisFolder["file"])
                    {
                        if (lastFolder != null)
                        {
                            t= lastFolder.SelectToken(file.Path.Replace("digest.folder.", ""));
                            
                        }
                        UpdateFile(downloadurl, folder, file, t, job);
                    }

                }
                else
                {
                    if (lastFolder != null)
                    {
                        t = lastFolder.SelectToken(thisFolder.Path.Replace("digest.folder.", ""));
                        t = t["file"];
                        
                    }
                    UpdateFile(downloadurl, folder, thisFolder["file"], t, job);
                }
            }
            
            if (thisFolder["folder"] != null)
            {
                if (thisFolder["folder"].Type == JTokenType.Array)
                {
                    foreach (JToken jsonfolder in thisFolder["folder"])
                    {
                        if (jsonfolder["@name"] == null)
                        {
                            UpdateFilesInFolder(downloadurl, folder, jsonfolder, lastFolder, job);
                        }
                        else
                        {
                            string name = (string)jsonfolder["@name"];
                            UpdateFilesInFolder(downloadurl, folder + name + "/", jsonfolder, lastFolder, job);
                        }
                    }
                }
                else
                {
                    thisFolder = thisFolder["folder"];
                    if (thisFolder["@name"] == null)
                    {
                        UpdateFilesInFolder(downloadurl, folder, thisFolder["folder"], lastFolder, job);
                    }
                    else
                    {

                        string name = (string)thisFolder["@name"];
                        UpdateFilesInFolder(downloadurl, folder + name + "/", thisFolder["folder"], lastFolder, job);
                    }
                }
            }
            
        }

        private static void UpdateFile(string downloadurl, string folder, JToken file, JToken oldFile, string job)
        {
            if (file["@sha"] == null) return;
            
            string shahash = (string)file["@sha"];
            string filename = (string)file["@name"];


            //If oldFile is null, that means this is a new file.
            if (oldFile != null)
            {
                string oldsha = (string)oldFile["@sha"];
                
                if (shahash == oldsha)
                {
                    return;
                }
            }
            

            
           

            shahash = shahash.Insert(2, "/").Insert(6, "/");

            int uncompressedsize = (int)file["@uncompressedSize"];
            int compressedsize = (int)file["@compressedSize"];

            if (uncompressedsize > compressedsize)
            {              
                DownloadAndDecompress(filename, downloadurl + "/" + shahash, folder + filename, job);               
            }
            else
            {
                Download(filename, downloadurl + "/" + shahash, folder + filename, job);
            }

            Console.WriteLine("Downloading: " + filename);


        }

        private static void Download(string filename, string url, string outputfilename, string job)
        {
            DoDownloadJob(new DownloadJob()
            {
                url = url,
                output = outputfilename,
                decompress = false,                
                name = filename,
                status = filename + " Has Not Started",
                jobtag = job,
            });
        }

        private static void DownloadAndDecompress(string filename, string url, string outputfilename, string job)
        {
            DoDownloadJob(new DownloadJob()
            {
                url = url,
                output = outputfilename,
                decompress = true,                
                name = filename,
                status = filename + " Has Not Started",
                jobtag = job
            });

            

        }

        private static void DoDownloadJob(DownloadJob job)
        {
            DownloadQueue.Enqueue(job);
        }

       

        public static int DownloadJobsToComplete()
        {
            return DownloadQueue.Count;
        }

        public static string[] ReportDownloadStatus()
        {
            List<string> s = new List<string>();
            s.Add("Files In Queue: " + DownloadQueue.Count);
            Parallel.ForEach(DownloadingQueue, j =>
                {
                    if (j == null) return;
                    s.Add(j.status);
                });
            
            return s.ToArray();
        }

        public static JObject GetJobjectFromManifest(string Manifest)
        {

            WebClient c = new WebClient();
            string d = c.DownloadString(Manifest + ".txt");

            System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
            doc.LoadXml(d);

            string jsonText = JsonConvert.SerializeXmlNode(doc, Formatting.Indented);
            return JObject.Parse(jsonText);
        }

        const int MaxFilesToDownload = 1;
        static int FilesBeingDownloaded = 0;

        static object CountLock = new object();
        public static void DownloadThreadWork()
        {
            WebClient cl = new WebClient();

            cl.Headers.Add("user-agent", "Quicksilver Player/1.0.3.183 (Windows; PlanetSide 2)");

            cl.DownloadProgressChanged += new DownloadProgressChangedEventHandler(cl_DownloadProgressChanged);
            cl.DownloadDataCompleted += new DownloadDataCompletedEventHandler(cl_DownloadDataCompleted);

            while (true)
            {

                lock (CountLock)
                {
                    if (FilesBeingDownloaded < MaxFilesToDownload)
                    {
                        DownloadJob job;
                        if (DownloadQueue.TryDequeue(out job))
                        {
                            
                            
                            cl.DownloadDataAsync(new Uri(job.url), job);
                            FilesBeingDownloaded++;
                        }
                        DownloadingQueue.Add(job);
                    }
                }
               
                Thread.Sleep(100);
            }
            
        }

        static void cl_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e)
        {
            DownloadJob job = (DownloadJob)e.UserState;
            if (e.Cancelled)
            {
                job.status = job.name + " Cancelled";
                Console.WriteLine("Job was Cancelled!");

                return;
            }
            if (e.Error != null)
            {
                job.status = job.name + " ERROR";
                Console.WriteLine("Error: " + e.Error);
                return;
            }


            byte[] data = e.Result;
            if (job.decompress)
            {
                job.status = job.name + " is Decompressing";
                data = SevenZip.Compression.LZMA.SevenZipHelper.Decompress(data);
            }

            using (BinaryWriter wr = new BinaryWriter(File.Open(job.output, FileMode.OpenOrCreate)))
            {
                job.status = job.name + " is Writing";
                if (data == null)
                {

                    job.status = job.name + " NULL DATA";
                    return;
                }

                wr.Write(data);
            }
            job.status = job.name + " is Waiting on lock";

            lock (CountLock)
            {
                int jobsLeft = DownloadQueue.Where(s => s.jobtag == job.jobtag).Count();

                if (jobsLeft == 0)
                {
                    JobCallbacks[job.jobtag](job.jobtag);
                    FullDownloadJobComplete d;
                    JobCallbacks.TryRemove(job.jobtag, out d);
                    
                }

                FilesBeingDownloaded--;
                DownloadJob j;
                DownloadingQueue.Remove(job);
            }

            job.status = job.name + " Done";

            Console.WriteLine(job.name + " is Done");
        }

        static void cl_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e)
        {
            DownloadJob j = (DownloadJob)e.UserState;
            j.status = string.Format("Downloading {0}: {1}% ({2}/{3})", j.name, e.ProgressPercentage, e.BytesReceived, e.TotalBytesToReceive);

        }
     }
}