Script to list dependencies and license from nuget

list-dependencies.fsx

//asdasda
#load @"C:/Utilities/fsharp-iteractive/load-all.fsx"
open System.ComponentModel
//#load @"C:\Utilities\fsharp-iteractive\packages\FsLab\FsLab.fsx"
#r "FSharp.Data.dll"
#r "System.Xml.dll"
#r "System.Xml.Linq.dll"
#r "FsPickler.dll"
#r "FsPickler.Json.dll"

open System.IO

let [<Literal>] target = __SOURCE_DIRECTORY__ + @"\package-license.json"

open FSharp.Data
open System.IO
open System.Text.RegularExpressions
open Microsoft.FSharp.Control
open System
open System.Net
open System.Runtime.Remoting


type License =
 | GNU2 //GNU GENERAL PUBLIC LICENSE Version2
 | Apache2
 | MIT
 | MsPL //Microsoft Public License (https://github.com/GregFinzer/Compare-Net-Objects/wiki/Licensing)
 | GNUlgpl // LGNU LESSER GENERAL PUBLIC LICENSE
 | MITX11
 | Microsoft // MICROSOFT SOFTWARE LICENSE TERMS
 | BSD
 | NewBSD
 | Other


type UnresolvedRemoteData = { Title : string; LicenseUrl : string option; ProjectUrl : string option }
type ResolvedRemoteData = {
         Title : string;  
         LicenseUrl  : string option; 
         ProjectUrl : string option; 
         License : Result<License, string> }

type ResolvedData = { Title : string; Uri : string list; License : License }

type PackageData = Cached of ResolvedData | Remote of Result<ResolvedRemoteData, string> 
type PackageInformation = { Id : string; Data : PackageData }


let (|Regex|_|) pattern content  = 
    match System.Text.RegularExpressions.Regex.Match(content, pattern) with 
    | x  when x.Success -> [for g in x.Groups -> g.Value] |> Some
    | _ -> None


let predefinedLicenses = function   
| "Newtonsoft.Json" -> Some { Title = "NEWTOSOFT.JSON"; License = MIT; Uri = [ "https://raw.githubusercontent.com/JamesNK/Newtonsoft.Json/master/LICENSE.md"; ] }
| "log4net" -> Some { Title = "LOG4NET"; License = Apache2; Uri = [ "http://logging.apache.org/log4net/license.html"; ] }
| "Microsoft.Tpl.Dataflow" -> Some { Title = "MICROSOFT TPL DATAFLOW"; License = Microsoft; Uri = [ "http://www.microsoft.com/net/dotnet_library_license.htm"; ] }
| Regex "Ninject.*" _ -> Some { Title = "NINJECT"; License = Apache2; Uri = [ "https://raw.githubusercontent.com/ninject/ninject/master/LICENSE.txt"; "https://raw.githubusercontent.com/ninject/ninject.extensions.conventions/master/LICENSE.txt";  "https://github.com/ninject/ninject.extensions.wcf/raw/master/LICENSE.txt"; ] }
| "PostSharp" -> Some { Title = "POSTSHARP EXPRESS"; License = Other; Uri = [ "https://www.postsharp.net/downloads/legal"; ] }
| Regex "Rx-.*" _ -> Some { Title = "MICROSOFT REACTIVE EXTENSIONS"; License = Microsoft; Uri = [ "https://msdn.microsoft.com/en-us/hh295787"; ] }
| "StackExchange.Redis.StrongName" -> Some { Title = "STACKEXCHANGE.REDIS"; License = MIT; Uri = [ "https://raw.githubusercontent.com/StackExchange/StackExchange.Redis/master/LICENSE"; ] }
| "protobuf-net" -> Some { Title = "PROTOBUF-NET"; License = BSD; Uri = [ "http://protobuf-net.googlecode.com/svn/trunk/Licence.txt"; ] }
| "protobuf-net-data" -> Some { Title = "PROTOBUF-NET-DATA"; License = Apache2; Uri = [ "https://raw.githubusercontent.com/rdingwall/protobuf-net-data/master/protobuf-net-data.license.txt"; ] }
| "IronPython" -> Some { Title = "IRONPYTHON"; License = Apache2; Uri = [ "https://ironpython.codeplex.com/license"; ] }
| "" -> Some { Title = "R.NET"; License = NewBSD; Uri = [ "https://rdotnet.codeplex.com/license"; ] }
| "jacobslusser.ScintillaNET.Signed" -> Some { Title = "SCINTILLA"; License = Other; Uri = [ "http://www.scintilla.org/License.txt"; ] }
| "Automatonymous" -> Some { Title = "Automatonymous"; License = Apache2; Uri = [ "https://github.com/MassTransit/Automatonymous/blob/develop/LICENSE" ] }
| "EntityFramework" -> Some { Title = "EntityFramework"; License = Microsoft; Uri = [ "http://go.microsoft.com/fwlink/?LinkID=262998"; "http://go.microsoft.com/fwlink/?LinkID=263480"] };
| "Microsoft.AspNet.WebApi.Client" -> Some { Title = "Microsoft.AspNet.WebApi.Client"; License = Microsoft; Uri = [ "http://www.microsoft.com/web/webpi/eula/net_library_eula_ENU.htm"; "https://www.asp.net/web-api"] };
| "Microsoft.Net.Http" -> Some { Title = "Microsoft.Net.Http"; License = Microsoft; Uri =  [ "http://go.microsoft.com/fwlink/?LinkId=329770"; "http://go.microsoft.com/fwlink/?LinkID=280055"] };
| _ -> None




// Acquire Package listing from current path
module PackageListing = 
    let [<Literal>] repositoriesPath = __SOURCE_DIRECTORY__ + @"\packages\repositories.config"
    let [<Literal>] samplePackagesConfig = __SOURCE_DIRECTORY__ + @"\ui\packages.config"
    type RepositoryList = XmlProvider<repositoriesPath>
    type PackagesConfig = XmlProvider<samplePackagesConfig>

    let listPackages = RepositoryList.Load(repositoriesPath).Repositories 
                        |> Seq.map (fun r -> __SOURCE_DIRECTORY__ + @"\pacakges\" + r.Path |> Path.GetFullPath)
                        |> Seq.toList
                        |> Seq.map PackagesConfig.Load
                        |> Seq.collect (fun p -> p.Packages)
                        |> Seq.map (fun p -> p.Id)
                        |> Seq.distinct
                        |> Seq.toList



//Download license links from Nuget
module NugetPackageRetrival = 
    type NugetPackageApi = JsonProvider<"https://api-v2v3search-0.nuget.org/query?q=Ninject&prerelease=false&take=1">
    let stringOption str = if String.IsNullOrEmpty str then None else Some (str)

    // captures package links from interenet
    let fetchPackageData package = 
        async { 
            let url = @"https://api-v2v3search-0.nuget.org/query?q=" + package + "&prerelease=false&take=1"
            let! result = NugetPackageApi.AsyncLoad url
            try
                return result.Data |> Seq.toList |> function 
                                                    | h::_  when String.Compare (h.Id2, package, true) <> 0 -> Error "Not exact package found" 
                                                    | h::_ ->  Ok { Title = h.Title;
                                                                            LicenseUrl = stringOption h.LicenseUrl; 
                                                                            ProjectUrl = stringOption h.ProjectUrl } 
                                                    | _ -> Error "Package Not Found on Nuget" 
            with error -> return Error error.Message 
        }

//resolve license type
module LicenseResolve = 
    let fetchUrlContent x =  async {  
                                let uri = Uri(x);
                                try 
                                    let! content = (new WebClient()).AsyncDownloadString(uri); 
                                    return Ok content            
                                with er -> return Error er.Message }

    let parseLicense content = 
        if (Regex.Match (content, @"GNU\sGENERAL\sPUBLIC\sLICENSE\s*Version\s2")).Success then Ok GNU2
        else if (Regex.Match (content, @"MIT License")).Success then Ok MIT
        else if (Regex.Match (content, @"Apache\sLicense[,\s]*Version\s2.0|Apache License 2.0|Apache License v2|www.apache.org/licenses/LICENSE-2.0", RegexOptions.Singleline)).Success then Ok Apache2
        else if (Regex.Match (content, @"Microsoft Public License")).Success then Ok MsPL
        else if (Regex.Match (content, @"GNU Lesser General Public License")).Success then Ok GNUlgpl
        else if (Regex.Match (content, @"MIT.X11")).Success then Ok MITX11
        else if (Regex.Match (content, @"MICROSOFT SOFTWARE LICENSE TERMS")).Success then Ok Microsoft
        else Error "Could not parse from url"

    let acquireLicenseFromUrl url = async { let! content = fetchUrlContent url; 
                                           return Result.bind parseLicense content} 

    
    let acquireLicenseFromRemoteData (pkgData :UnresolvedRemoteData) = 
        async {
            match pkgData with
            | { LicenseUrl = None;       ProjectUrl = None } -> return Error "Missing url information to retrieve license"
            | { LicenseUrl = Some lUrl;  ProjectUrl = None } -> return! acquireLicenseFromUrl lUrl
            | { LicenseUrl = None;       ProjectUrl = Some pUrl } -> return! acquireLicenseFromUrl pUrl
            | { LicenseUrl = Some lUrl;  ProjectUrl = Some pUrl } -> 
                let! l1 = acquireLicenseFromUrl lUrl
                match l1 with
                | Ok l -> return Ok l
                | Error er -> let! l2 = acquireLicenseFromUrl pUrl
                              match l2 with 
                              | Ok l -> return Ok l
                              | _ -> return Error er 
        }

    let resolveData (data:UnresolvedRemoteData) =  async {
        let! license = acquireLicenseFromRemoteData data
        return  { Title = data.Title;
                  LicenseUrl = data.LicenseUrl;
                  ProjectUrl = data.ProjectUrl;
                  License = license } }

// end of modules

type Microsoft.FSharp.Control.Async with    
     static member map f x = async.Bind(x, f >> async.Return) 
     static member bind f x = async.Bind(x, f) 


let getPackageRemote id =  
    async { 
        let! pkgdata = NugetPackageRetrival.fetchPackageData id
        match pkgdata with
        | Error r -> return Error r
        | Ok data -> let! resolved = LicenseResolve.resolveData data
                     return Ok resolved
    }    

let getPackage (licenseOverride) (id:string) : Async<PackageInformation>= 
    let data = match licenseOverride id with
               | Some d -> async.Return <| Cached d 
               | None -> Async.map Remote <| Async.bind getPackageRemote (async.Return id)
    Async.map (fun d -> { Id = id; Data = d }) data


let result = PackageListing.listPackages 
            |> Seq.map (getPackage predefinedLicenses)
            |> Async.Parallel
            |> Async.RunSynchronously


let titleIds = result |> Array.choose (function | { Id = id; Data = Remote (Ok { Title = title; LicenseUrl = _; ProjectUrl = _; License = _}) } -> Some (title, id)| _ -> None )

let failedCapture = result |> Array.choose (function 
                                | { Id = id; Data = Remote(Error er) } -> Some (id, er, []) 
                                | { Id = id; Data = Remote( Ok {Title = _; LicenseUrl = l1; ProjectUrl = l2; License = Error err } )} -> Some (id, err, [l1; l2]  ) 
                                | _ -> None )



result |> Array.choose (fun x -> match x with | { Id =  _; Data = Remote _ } ->  Some x | _ -> None )