Nipster

NpmBuilder(datatables).cs

using System;
using System.Data.Entity;
using System.Data.SqlTypes;
using System.Linq;

namespace Nipster
{
    public static class NpmBuilder
    {
        public static dynamic BuildNipsterNpmDataTableSerialized(NipsterContext nipsterContext)
        {
            var npms = nipsterContext.NpmPackages.Include(npm => npm.GitHubRepo).ToList();
            var count = npms.Count;
            var data = npms.Select(npm =>
            {
                var gitHubRepo = npm.GitHubRepo ?? new GitHubRepo();
                if (!gitHubRepo.Found) gitHubRepo = new GitHubRepo();

                var date = npm.Modified.HasValue ? npm.Modified.Value : SqlDateTime.MinValue.Value;
                return new object[]
                {
                    npm.Id,
                    gitHubRepo.Id,
                    npm.Description,
                    npm.AuthorUrl + ";" + npm.Author,
                    date.ToString("yyyy-MM-dd"),
                    gitHubRepo.Forks,
                    gitHubRepo.Stargazers,
                    gitHubRepo.Watchers,
                    npm.Keywords
                };
            }).ToArray();
            return new
            {
                lastUpdate = DateTime.UtcNow.ToString("u"),
                aaData = data
            };
        }
    }
}

NpmBuilder.cs

using System;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Shared.Protocol;
using Newtonsoft.Json;

namespace NipsterRole
{
    public class NpmBuilder : Worker
    {
        public NpmBuilder() : base(TimeSpan.FromHours(12), "npm")
        {
        }

        public override void Run()
        {
            Trace.TraceInformation("Npm builder: Building packages json-file");

            var build = Nipster.NpmBuilder.BuildNipsterNpmDataTableSerialized(NipsterContext);
            var json = JsonConvert.SerializeObject(build);
            var jsonGzipped = Zip(json);

            var storageAccount =
                CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));

            var client = storageAccount.CreateCloudBlobClient();

            Trace.TraceInformation("Npm builder: Forcing CORS");
            var properties = client.GetServiceProperties();
            var rule = properties.Cors.CorsRules.FirstOrDefault(r => r.AllowedOrigins.Contains("*"));
            if (rule == null)
            {
                properties.Cors.CorsRules.Add(new CorsRule
                {
                    AllowedOrigins = new[] {"*"},
                    MaxAgeInSeconds = (int) TimeSpan.FromDays(365).TotalSeconds,
                    AllowedMethods = CorsHttpMethods.Get
                });
                client.SetServiceProperties(properties);
            }

            Trace.TraceInformation("Npm builder: Uploading");
            var containerRef = client.GetContainerReference("cdn");
            var blobReference = containerRef.GetBlockBlobReference("npm-datatables.json");

            blobReference.Properties.ContentType = "application/json";
            blobReference.Properties.ContentEncoding = "gzip";
            blobReference.UploadFromByteArray(jsonGzipped, 0, jsonGzipped.Length);

            Trace.TraceInformation("Npm worker: Done building packages");
        }

        public static byte[] Zip(string str)
        {
            var bytes = Encoding.UTF8.GetBytes(str);

            using (var msi = new MemoryStream(bytes))
            using (var mso = new MemoryStream())
            {
                using (var gs = new GZipStream(mso, CompressionMode.Compress))
                {
                    msi.CopyTo(gs);
                }

                return mso.ToArray();
            }
        }
    }
}

NpmClient.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace Nipster
{
    public static class NpmClient
    {
        private const string NpmUrl = "http://registry.npmjs.org/-/all/since?startkey=";

        public static async Task<NpmResult> GetPackages(long lastUpdate)
        {
            using (var client = new HttpClient())
            {
                client.Timeout = new TimeSpan(0, 0, 10, 0);
                var response = await client.GetAsync(NpmUrl + lastUpdate);
                var data = await response.Content.ReadAsAsync<Dictionary<string, dynamic>>();
                var npmPackages = data.Where(kv => kv.Key != "_updated").Select(kv => kv.Value).Select(v =>
                {
                    var url = GitHubUrl(() => v.repository.url)
                              ?? GitHubUrl(() => v.repository[0].git)
                              ?? GitHubUrl(() => v.repository[0].url)
                              ?? GitHubUrl(() => v.repository.git)
                              ?? GitHubUrl(() => v.repository)
                              ?? GitHubUrl(() => v.homepage)
                              ?? GitHubUrl(() => v.bugs.url);
                    var gitHubName = GitHubName("" + url);

                    var authorUrl = NoThrow.String(() => v.author.url) ?? NoThrow.String(() => v.author.email);

                    return new NpmPackage
                    {
                        Id = NoThrow.String(() => v.name),
                        Description = NoThrow.String(() => v.description),
                        Keywords = string.Join(" ", NoThrow.StringArray(() => v.keywords) ?? new string[]{}),
                        Modified = NoThrow.DateTime(() => (DateTime)v.time.modified),
                        Author = NoThrow.String(() => v.author.name),
                        AuthorUrl = authorUrl,
                        GitHubRepoId = gitHubName
                    };
                });

                return new NpmResult
                {
                    LastUpdate = data["_updated"],
                    NpmPackages = npmPackages
                };
            }
        }

        private static string GitHubUrl(Func<dynamic> func)
        {
            var u = "" + NoThrow.String(func);
            return Regex.IsMatch(u, "github.com", RegexOptions.IgnoreCase) ? u : null;
        }

        private static string GitHubName(string url)
        {
            url = Regex.Replace(url, @".*github.com.", "", RegexOptions.IgnoreCase);
            url = Regex.Replace(url, @"\.git$", "", RegexOptions.IgnoreCase);
            url = Regex.Replace(url, @"(.*\/.*)\/issues", "", RegexOptions.IgnoreCase);
            return url;
        }
    }
}

NpmPusher.cs

using System;
using System.Diagnostics;
using System.Linq;
using Microsoft.WindowsAzure.Storage.Queue;
using Nipster;

namespace NipsterRole
{
    public class NpmPusher : Worker
    {
        public NpmPusher() : base(TimeSpan.FromHours(6), "npm")
        {
        }

        public override void Run()
        {
            Queue.FetchAttributes();
            if (Queue.ApproximateMessageCount > 0) return;

            Trace.TraceInformation("Npm pusher: Last npm client update: {0}", NipsterConfig.LastNpmUpdate);
            var task = NpmClient.GetPackages(NipsterConfig.LastNpmUpdate);
            task.Wait();
            var packages = task.Result;
            Trace.TraceInformation("Npm pusher: Last update: {0}", packages.LastUpdate);

            NipsterConfig.LastNpmUpdate = packages.LastUpdate;
            NipsterContext.SaveChanges();

            Trace.TraceInformation("Npm pusher: Got {0} packages", packages.NpmPackages.Count());
            packages.NpmPackages.ToList().ForEach(npm =>
            {
                Trace.TraceInformation("Npm pusher: Package {0}", npm.Id);
                Queue.AddMessage(new CloudQueueMessage(Newtonsoft.Json.JsonConvert.SerializeObject(npm)));
            });
        }
    }
}

NpmWorker.cs

using System;
using System.Data.Entity.Migrations;
using System.Diagnostics;
using System.Linq;
using Microsoft.WindowsAzure.Storage.Queue;
using Nipster;

namespace NipsterRole
{
    public class NpmWorker : MyHappyLittleWorker
    {
        public NpmWorker() : base(TimeSpan.FromSeconds(10), "npm")
        {
        }

        public override bool Run(CloudQueueMessage message)
        {
            var npmPackage = message.As<NpmPackage>();
            Trace.TraceInformation("Npm: Package {0} with GitHub {1}", npmPackage.Id, npmPackage.GitHubRepoId);
            var gh = GitHub(npmPackage.GitHubRepoId);

            npmPackage.GitHubRepo = gh;
            npmPackage.GitHubRepoId = null;

            NipsterContext.NpmPackages.AddOrUpdate(npm => npm.Id, npmPackage);
            NipsterContext.SaveChanges();

            return true;
        }

        private GitHubRepo GitHub(string gitHubRepoId)
        {
            if (string.IsNullOrEmpty(gitHubRepoId)) return null;

            var gh = NipsterContext.GitHubRepos.FirstOrDefault(g => g.Id == gitHubRepoId) ??
                     NipsterContext.GitHubRepos.Add(new GitHubRepo {Id = gitHubRepoId});
            NipsterContext.SaveChanges();

            return gh;
        }
    }
}