Code to allow simple automated access to the Thomson-Reuters Eikon API using F#

Thomson-Reuters Eikon API.fsx

//#load "packages/FsLab/FsLab.fsx"
// !!!! CHECK 32 BIT ONLY!!!
// Tools > Options > F# Tools > F# Interactive 
// https://developers.thomsonreuters.com/sites/default/files/Eikon_for_Developers_09032016.pdf

(**
Scripting Eikon time-series (.NET) API 
======================================

Developing some code to allow simple automated access to the Thomson-Reuters Eikon API.

Goals
----------
*   Develop some code to wrap the imperative style of the API to permit a more functional approach
*   Use [GooglePlot](googleplot) & [Plotly](plotly) charting frameworks with the help of the [XPlot](xplot) framework.
*)

(**
## Load, include & reference script, folders & libraries
For interactive files it is important to explicitly itemize library files, rather than having references be handed by the IDE.

## Instructions 
Instructions for use in Visual Studio
* Install FsLab templates: Tools > Extensions > Search for "FsLab" & install it
* Create FsLab solution. File > New Project > FsLab Journal
* Add to paket.dependencies:
    - ThomsonReuters.Desktop.SDK.DataAccess.Signed.x64 
    - WindowsBase 
* Add to References
    - Common.Logging
    - Newtonsoft.Json
    - ThomsonReuters.Udap.BusTools
    - ThomsonReuters.Udap.Ipc.Managed.Common
    - ThomsonReuters.Udap.ManagedPS
* Right-click References. Generate references for F# interactive, to create file Scripts/load-references-debug.fsx

*)

(*** hide ***)
// For most refs, add to project References & re-generate file
#I __SOURCE_DIRECTORY__
#r "packages/Common.Logging/lib/net40/Common.Logging.dll"
#r "packages/Common.Logging.Core/lib/net40/Common.Logging.Core.dll"
#r "packages/ThomsonReuters.Desktop.SDK.DataAccess.Signed.x64/lib/net40-Client/ThomsonReuters.Desktop.SDK.DataAccess.dll"
#r "packages/ThomsonReuters.Udap.Ipc.Signed.x64/lib/net40-Client/ThomsonReuters.Udap.BusTools.dll"
#r "packages/ThomsonReuters.Udap.Ipc.Signed.x64/lib/net40-Client/ThomsonReuters.Udap.Ipc.Managed.Common.dll"
#r "packages/ThomsonReuters.Udap.Ipc.Signed.x64/lib/net40-Client/ThomsonReuters.Udap.ManagedPS.dll"
#r "packages/Newtonsoft.Json/lib/net45/Newtonsoft.Json.dll"
#r "packages/protobuf-net/lib/net40/protobuf-net.dll"
#r "packages/Paket/tools/paket.exe"
#r "packages/Paket.Core/lib/net45/Paket.Core.dll"
#r "packages/Deedle/lib/net40/Deedle.dll"
#r "System.dll"
#r "System.Core.dll"
#r "System.Numerics.dll"
#r "System.Xml.dll"
#r "System.Windows.dll"
#r "System.Windows.Forms.dll"
#r "WindowsBase.dll"
#I "packages/ThomsonReuters.Udap.Ipc.Signed.x64/Tools"  // For EikonPipeDll etc.
#load "packages/FsLab/FsLab.fsx" 

(**
## Open libraries
*)
open System
open System.Windows.Threading   // DispatcherFrame (WindowsBase.dll)
open System.Collections.Generic
open Common.Logging
open ThomsonReuters.Desktop.SDK.DataAccess
open ThomsonReuters.Desktop.SDK.DataAccess.TimeSeries
open Deedle
open FSharp.Data
open FSharp.Charting
open RProvider
open RProvider.graphics
open RProvider.ggcorrplot
open RProvider.igraph
open XPlot.Plotly
open XPlot.GoogleCharts
open Paket
open MathNet.Numerics.LinearAlgebra
open MathNet.Numerics.LinearAlgebra.Double
open MathNet.Numerics.Statistics

(*** hide ***)
// Code for interacting the Paket references manager
let skipThis () =
    let (dependencies: Paket.Dependencies) = Paket.Dependencies.Locate(__SOURCE_DIRECTORY__)
    dependencies.GetInstalledPackages()

(**
## Module EikonTimeSeries (ETS)
The module has functions to:
* Launch an Eikon time-series data service
* Pull data from the data service for a particular Reuters Instrument Code (RIC)
* Do various things with the time series data that is provided by the Eikon API:
    - Build a Deedle Series for a given field e.g. "CLOSE"
    - Build a Deedle Frame for all fields e.g. "OPEN","CLOSE","VOLUME"
    - Print the data
* Print the name, description & supported intervals (periodicities) for the various views for a RIC
*)
module ETS = 
    type ReutersInstrumentCode = string       // https://en.wikipedia.org/wiki/Reuters_Instrument_Code
    type ApplicationCode = string
    // Function that sets all parameters for an ITimeSeriesDataRequestSetup object
    type SetDataRequestParameters= ITimeSeriesDataRequestSetup -> ITimeSeriesDataRequestSetup 
     
    let launchTimeSeriesDataService (ac:ApplicationCode): ITimeSeriesDataService  =
        let dsInstance = DataServices.Instance
        dsInstance.Initialize(ac)
        dsInstance.TimeSeries 

    let getTimeSeries (tsds:ITimeSeriesDataService) (setRequestParameters:SetDataRequestParameters) (ric:ReutersInstrumentCode) : seq<IData>  =
        let mutable records = Seq.empty<IData>   // IData is seq<KeyValuePair>  
        let frame = DispatcherFrame() 
        let dataReceivedCallback (dataChunk:DataChunk) =
            records <- Seq.append records dataChunk.Records 
            frame.Continue <- dataChunk.IsLast |> not  
            () 
        (tsds.SetupDataRequest(ric)
            |> setRequestParameters)  // apply the function to the TimeSeriesDataRequestSetup object
            //.WithFields([]).From(Nullable<DateTime> DateTime.Now).WithAdjustedPrice(true).WithNumberOfPoints(10).WithView("BID")
            .OnDataReceived(Action<DataChunk> dataReceivedCallback)  
            .CreateAndSend() 
            |> ignore
        Dispatcher.PushFrame(frame)  // "Enters an execute loop"         
        records   
    
    // The following functions all act on time-series data from the TimeSeriesDataService
    let toSeries (field:string) (data:seq<IData>) : Series<DateTime,obj> = 
        data 
        |> Seq.map (fun (r:IData) -> (r.Timestamp.Value, r.AllFields.[field] )) // |> unbox<float> - to cast to spECI"fic type )
        |> Series.ofObservations // seq<DateTime*obj>->Series<DateTime,obj>
    
    // Put all fields for a single RIC together in a frame e.g. ["CLOSE"; "VOLUME"]
    let toFrame (data:seq<IData>) : Frame<DateTime,string> = 
        data |> Seq.head 
        |> (fun r -> r.Keys |> Seq.filter (fun z-> z <> "TIMESTAMP")) // Seq of fields
        |> Seq.map (fun field -> 
            //printfn "%A" field
            (field, data 
                    |> Seq.map (fun r -> (r.Timestamp.Value, (r.AllFields.TryGetValue(field) |> OptionalValue.ofTuple))) 
                    |> Series.ofObservations ))
        |> Frame.ofColumns

    let printRecords (data : seq<IData>) : unit = 
        data |> Seq.iter (fun (r:IData) -> r.AllFields |> Seq.iter (fun kv -> printfn "%A - %A" kv.Key kv.Value))
    
    // Find all views (with supported intervals) for a given RIC
    // https://community.developers.thomsonreuters.com/questions/12560/withview-format.html
    let printViews (tsds:ITimeSeriesDataService) (ric:ReutersInstrumentCode)  : unit =
        let intervalsToTupleList (intervals:seq<Interval>) = intervals |> Seq.map (fun i ->  (i.Type,i.Length )) |> Seq.toList
        let viewsCallback (views:seq<View>) : unit = 
            views 
            |> Seq.iter (fun v-> printfn "%A, %A, %A" v.Name v.LongDescription (v.Intervals |> intervalsToTupleList|> ignore); (* Need to loop over interval types & lengths: v.Intervals *) )
        tsds.GetViewList(ric, Action<seq<View>> viewsCallback, null) |> ignore
        ()

open ETS

(**
// # Test the features of the Eikon Time Series (ETS) 
// ## (1) Launch the Eikon time-series data service
*)
let tsds = ETS.launchTimeSeriesDataService "EikonAPIApplication"

(**
// ## Find all views (with supported intervals) for a given RIC
*)
let bmoViews = ETS.printViews tsds "BMO.TO"

(**
// ## Function to set parameters of TimeSeriesDataRequestSetup
All parameters for the `TimeSeriesDataRequestSetup` object are supplied in the form of a function.
*)
let setRequestParasDflt: (ITimeSeriesDataRequestSetup -> ITimeSeriesDataRequestSetup) = 
    (fun z -> z.WithView("BID").WithFields([]).WithNumberOfPoints(Nullable<int>(100)))
    // .From(Nullable<DateTime>(DateTime.Now.AddYears(-1))).To(Nullable<DateTime>(DateTime.Now))
let testTSDRS = tsds.SetupDataRequest("BMO.TO") |> setRequestParasDflt 

(**
// ## Examples of getting & printing data for BMO.
*)
"BMO.TO" 
|> ETS.getTimeSeries tsds (fun z -> z.WithNumberOfPoints(Nullable<int>(3))) |> ETS.printRecords
let (bmoRecords0:seq<IData>) = ETS.getTimeSeries tsds id "BMO.TO"
let bmoCds =  "BMO10YUSAR=R" |> ETS.getTimeSeries tsds id  |> ETS.printRecords

"BMO10YUSAR=R" 
|> ETS.getTimeSeries tsds id  
|> ETS.printRecords

(**
// ## (2) Line chart for BMO
*)
(*** define-output:bmochart ***)
"BMO.TO" 
|> ETS.getTimeSeries tsds (fun z -> z.WithNumberOfPoints(Nullable<int>(252*3))) 
|> ETS.toSeries "CLOSE" 
|> Series.observations 
|> Seq.map (fun (k,v) -> k, v |> unbox<float>) 
|> FSharp.Charting.Chart.Line
(*** include-it:bmochart ***)


(**
// ## Create Deedle frame for all fields
*)
let bmoFrame = "BMO.TO" |> ETS.getTimeSeries tsds id  |> ETS.toFrame
(*** include-value:bmoFrame ***)

(**
// ## (3) The big five banks
*)

let topFiveBankRics = ["BMO.TO";"BNS.TO";"CM.TO";"RY.TO";"TD.TO";]

let ricToSeries (ric:ETS.ReutersInstrumentCode) =
    ric
    |> ETS.getTimeSeries tsds id 
    |> ETS.toSeries "CLOSE"
(**
Test it
*)
ricToSeries "BMO.TO"
(**
Get price histories
*)
let bigFiveBanksPrices:Frame<DateTime,string> =
    ["BMO.TO";"BNS.TO";"CM.TO";"RY.TO";"TD.TO";]
        |> List.map (fun ric -> printfn "%A" ric; (ric, ric |> ETS.getTimeSeries tsds id |> ETS.toSeries "CLOSE"))
        |> Frame.ofColumns
 (**
(4) Chart the prices
*)
let combinedPriceChart =
    ["BMO.TO";"BNS.TO";"CM.TO";"RY.TO";"TD.TO";]
        |> List.map (fun ric -> 
            ric 
            |> ETS.getTimeSeries tsds (fun z -> z.WithNumberOfPoints(Nullable<int>(252*5))) 
            |> ETS.toSeries "CLOSE" 
            |> Series.observations 
            |> Seq.map (fun (k,v)-> k,v|>unbox<float>)
            |> (fun (obs:seq<DateTime*float>) -> FSharp.Charting.Chart.Line(obs)))
        |> FSharp.Charting.Chart.Combine 
(*** include-value:combinedChart ***)

(**
## (5) Get histories of adjusted returns
*)
"BMO.TO" |> ETS.getTimeSeries tsds (fun z->z.WithView("TRTN"))
// "TRTN" "1 Day Gross Total Return"
let bigFiveBanksReturns:Frame<DateTime,string> =
    ["BMO.TO";"BNS.TO";"CM.TO";"RY.TO";"TD.TO";]
        |> List.map (fun ric -> printfn "%A" ric; (ric, ric 
                                                        |> ETS.getTimeSeries tsds (fun z->z.WithView("TRTN").WithNumberOfPoints(Nullable<int>(252*5))) 
                                                        |> ETS.toSeries "VALUE"))
        |> Frame.ofColumns
(**
(6) Chart the returns
*)
let combinedReturnChart =
    ["BMO.TO";"BNS.TO";"CM.TO";"RY.TO";"TD.TO";]
        |> List.map (fun ric -> 
            ric 
            |> ETS.getTimeSeries tsds (fun z->z.WithView("TRTN").WithNumberOfPoints(Nullable<int>(50))) 
            |> ETS.toSeries "VALUE" 
            |> Series.observations 
            |> Seq.map (fun (k,v)-> k,v|>unbox<float>)
            |> (fun (obs:seq<DateTime*float>) -> FSharp.Charting.Chart.Bar(obs)))
        |> FSharp.Charting.Chart.Combine 

(* (7) Scatter plot matrix for returns for Big 5 *)
R.plot(bigFiveBanksReturns)
 


let bigFiveCorr =
    bigFiveBanksReturns 
    |> Frame.fillMissingWith 0.0
    |> Frame.toMatrix
    |> Matrix.toColArrays
    |> Correlation.PearsonMatrix

R.ggcorrplot(bigFiveCorr|> Matrix.toFrame)

let canadaRics = 
    //[ ".GSPTSE"; 
    [".SPTSE"; ".SPTSEM"; ".SPTSES"; ".SPTTEN"; ".SPTTFS"; ".SPTTGD"; ".SPTTTK"; ".GTSX55"; ".GTSX50"; 
      ".GTSX60"; ".GSPTTMT"; ".GSPTTRE"; ".GSPTTIN"; ".GSPTTHC"; ".GSPTTUT"; ".GSPTTTS"; ".GSPTTCD"; ".GSPTTCS"; 
      ".GSPRTRE"; ".GSPTXGM"; ".GSPRTCM"; ".SPTSECP3"; ".GTSX2030"; ".TRGTSX60"; ".NTRGTSX60"; ".GTSX203020"; 
      ".GTSX551030"; ".GTSX403010"; ".GTSX452010"; ".GTSX452030"; ".GTSX20302010"; ".GTSX10101010"; ".GTSX40201040"; 
      ".GTSX25503020"; ".GTSX20304010" ]

let combinedPriceChart2 =
    canadaRics //|> List.take 5
    |> List.map (fun ric -> 
        ric 
        |> ETS.getTimeSeries tsds (fun z -> z.WithNumberOfPoints(Nullable<int>(252*5))) 
        |> ETS.toSeries "CLOSE" 
        |> Series.observations 
        |> Seq.map (fun (k,v)-> k,v|>unbox<float>)
        |> (fun (obs:seq<DateTime*float>) -> FSharp.Charting.Chart.Line(obs)))
    |> FSharp.Charting.Chart.Combine 

let canadaEqPrices:Frame<DateTime,string> =
    canadaRics 
    //|> List.take 5
    |> List.map (fun ric -> printfn "%A" ric; (ric, ric 
                                                    |> ETS.getTimeSeries tsds (fun z->z.WithView("BID").WithNumberOfPoints(Nullable<int>(50))) 
                                                    |> ETS.toSeries "CLOSE"))
    |> Frame.ofColumns

let canadaEqReturns:Frame<DateTime,string> =
    canadaRics 
    |> List.take 5
    |> List.map (fun ric -> printfn "%A" ric; (ric, ric 
                                                    |> ETS.getTimeSeries tsds (fun z->z.WithView("TRTN").WithNumberOfPoints(Nullable<int>(252*1))) 
                                                    |> ETS.toSeries "VALUE"))
    |> Frame.ofColumns

// #region

(*** Hide ***)
(**
The simplest implementation just uses the methods from the API. More sophisticated approaches store the parameters in a
record or map, & then work through the provided parameters applying them one-by-one using the methods of the API. 
*)

// From the Cross Economic Data Excel template
let cadEcoIndicatorRics = 
    [ "CABOCR=ECI"; "CABUD=ECI"; "CABUDY=ECI"; "CABPER=ECI"; "CACURA=ECI"; "CACAPU=ECI"; "CACPIX=ECI"; "CACPXX=ECI"; 
      "CACPMM=ECI"; "CACPI=ECI"; "CACPIY=ECI"; "CAEXP=ECI"; "CAGDPM=ECI"; "CAGDPA=ECI"; "CAGDP=ECI"; "CAHSTA=ECI"; 
      "CAGDPD=ECI"; "CAIMP=ECI"; "CAIPMI=ECI"; "CAEMPC=ECI"; "CALPR=ECI"; "CAMFGS=ECI"; "CAHPRI=ECI"; "CAPPI=ECI"; 
      "CAPPIY=ECI"; "CARAW=ECI"; "CARAWY=ECI"; "CAXAUT=ECI"; "CARSLS=ECI"; "CAISEC=ECI"; "CAISEF=ECI"; "CATBAL=ECI"; 
      "CAUNR=ECI"; "CAWTRA=ECI"; "CAIVSA=ECI"; "CAPMIM=ECI"; "CAPRT=ECI"; "CAEMPF=ECI"; "CAEMPT=ECI" ]

"CABOCR=ECI" |> ETS.getTimeSeries tsds id |> ETS.printRecords

(*** Hide ***)
let viewMap =               // Some views for equities
    [ ("ASK", "Ask")
      ("BID", "Bid")
      ("BLKVOLUM", "Block Volume")
      ("MID_PRICE", "Mid Price")
      ("MTD_TRTN", "Month To Date Gross Total Return")
      ("NDA_RAW", "NDA_RAW")
      ("NETCHNG_1", "Net Change")
      ("PCTCHNG", "Percentage Change")
      ("QTD_TRTN", "Quarter To Date Gross Total Return")
      ("TRDPRC_1", "Trade Price")
      ("TRNOVR_UNS", "Turnover")
      ("TRTN", "1 Day Gross Total Return")
      ("TRTN_1M", "1 Month Gross Total Return")
      ("TRTN_1W", "1 Week Gross Total Return")
      ("TRTN_2Y", "2 Years Gross Total Return")
      ("TRTN_3MT", "1 Quarter Gross Total Return")
      ("TRTN_3Y", "3 Years Gross Total Return")
      ("TRTN_4Y", "4 Years Gross Total Return")
      ("TRTN_5Y", "5 Years Gross Total Return")
      ("TSDB_RAW", "TSDB_RAW")
      ("TSFCP_RAW", "TSFCP_RAW")
      ("VWAP", "VWAP")
      ("YR_TRTN", "1 Year Gross Total Return")
      ("YTD_TRTN", "Year To Date Gross Total Return") ]
    |> Map.ofList

(**
Test it
*)

//Plotly.Signin(Credentials.username, Credentials.key)
let skipThis () =
    XPlot.Plotly.Credentials.username <- "xxxxxxx"
    XPlot.Plotly.Credentials.key <- "xxxxxxx"
    XPlot.Plotly.Plotly.Signin("xxxxxxx","xxxxxxxxx")
    ()
(**
## combinedChartXP
*)
let combinedChartXP =
    ["BMO.TO";"BNS.TO";"CM.TO";"RY.TO";"TD.TO";]
        |> List.map (fun ric -> 
            ric 
            |> ricToSeries 
            |> Series.observations 
            |> Seq.map (fun (k,v)-> k,v|>unbox<float>)
            |> (fun (obs:seq<DateTime*float>) -> XPlot.Plotly.Graph.Line(obs)))
        |> FSharp.Charting.Chart.Combine 

(**
## Plotly
*)
let trace1 =
    XPlot.Plotly.Graph.Scatter(
        x = [1; 2; 3; 4],
        y = [10; 15; 13; 17]
    )

let trace2 =
    XPlot.Plotly.Graph.Scatter(
        x = [2; 3; 4; 5],
        y = [16; 5; 11; 9]
    )

[trace1; trace2]
|> XPlot.Plotly.Graph.Line
|> XPlot.Plotly.Graph.WithWidth 700
|> XPlot.Plotly.Graph.WithHeight 500