fpml_example.jl

fpml_example.jl

# On discourse.julialang.org, Lincoln Hannah asked if we support XML.
# We did support XML in an earlier version of DataKnots, but it requires
# "IndexVector" which we removed to make a minimal, documented release.
# (https://github.com/rbt-lang/DataKnots.jl/blob/d7488cc31d06bbdd2d0b6129784994a39ce72dd7/src/combinators/xml.jl
# https://github.com/rbt-lang/DataKnots.jl/blob/d7488cc31d06bbdd2d0b6129784994a39ce72dd7/src/queries/xml.jl)
#
# One could still query XML, but you have to first load it into an
# in-memory dictionary.  Moreover, you have to define your schema via
# a query. Here is an example of how one could do this with FpML, a
# financial product markup language.

using DataKnots, XMLDict, Dates
import Base: getproperty
import DataKnots: lookup

# Lookup function compatible with `XMLDict` output
function lookup(ity::Type{<:XMLDict.OrderedDict}, name::Symbol) 
    oty = Union{Any, Missing}
    key = string(name)
    if startswith(key, "@")
        key = Symbol(key[2:end])
    end
    DataKnots.lift(get, key, missing) |> 
      DataKnots.designate(ity, oty |> DataKnots.IsLabeled(name))
end
Base.getproperty(nav::DataKnots.Navigation, s::String) = 
    Base.getproperty(nav, Symbol(s))

# When building queries, for them to be understood by DataKnots,
# you have to assert the datatype of the elements.
IsDict = Is(XMLDict.OrderedDict)

# XMLDict assumes child elements are singular, unless there is two
# or more, then it's plural. We can normalize this.
vectorize(X::Any) = [X]
vectorize(X::Vector) = X

# Here is our example XML document...
xml_string  = ("""
    <Portfolio>

		<Trade>

			<Header>
			  <tradeID>10001</tradeID>
			  <tradeDate>2020-01-30</tradeDate>
			  <book>FX_Book</book>
			</Header>

			<product>FX_Forward</product>

			<fxLeg>
			  <exchangedCurrency1>
				<payerPartyReference id="THIS_BANK" />
				<receiverPartyReference id="OTHER_BANK_1" />
				<paymentAmount>
				  <currency>USD</currency>
				  <amount>1100000</amount>
				</paymentAmount>
			  </exchangedCurrency1>

			  <exchangedCurrency2>
				<payerPartyReference id="OTHER_BANK_1" />
				<receiverPartyReference id="THIS_BANK" />
				<paymentAmount>
				  <currency>EUR</currency>
				  <amount>1000000</amount>
				</paymentAmount>
			  </exchangedCurrency2>

			  <paymentDate>2020-05-06</paymentDate>
			</fxLeg>

		</Trade>

		<Trade>

			<Header>
			  <tradeID>10002</tradeID>
			  <tradeDate>2020-02-28</tradeDate>
			  <book>FX_Book</book>
			</Header>

			<product>FX_Forward</product>

			<fxLeg>
			  <exchangedCurrency1>
				<payerPartyReference id="OTHER_BANK_2" />
				<receiverPartyReference id="THIS_BANK" />
				<paymentAmount>
				  <currency>USD</currency>
				  <amount>1200000</amount>
				</paymentAmount>
			  </exchangedCurrency1>

			  <exchangedCurrency2>
				<payerPartyReference id="THIS_BANK" />
				<receiverPartyReference id="OTHER_BANK_2" />
				<paymentAmount>
				  <currency>GBP</currency>
				  <amount>1000000</amount>
				</paymentAmount>
			  </exchangedCurrency2>

			  <paymentDate>2020-08-06</paymentDate>
			</fxLeg>

		</Trade>

		<Trade>

			<Header>
			  <tradeID>10003</tradeID>
			  <tradeDate>2020-03-30</tradeDate>
			  <book>FX_Book</book>
			</Header>

			<product>FX_Swap</product>

			<fxLeg>
			  <exchangedCurrency1>
				<payerPartyReference id="OTHER_BANK_3" />
				<receiverPartyReference id="THIS_BANK" />
				<paymentAmount>
				  <currency>USD</currency>
				  <amount>1200000</amount>
				</paymentAmount>
			  </exchangedCurrency1>

			  <exchangedCurrency2>
				<payerPartyReference id="THIS_BANK" />
				<receiverPartyReference id="OTHER_BANK_3" />
				<paymentAmount>
				  <currency>CAD</currency>
				  <amount>1000000</amount>
				</paymentAmount>
			  </exchangedCurrency2>

			  <paymentDate>2020-12-01</paymentDate>
			</fxLeg>


			<fxLeg>
			  <exchangedCurrency1>
				<payerPartyReference id="THIS_BANK" />
				<receiverPartyReference id="OTHER_BANK_3" />
				<paymentAmount>
				  <currency>USD</currency>
				  <amount>1200000</amount>
				</paymentAmount>
			  </exchangedCurrency1>

			  <exchangedCurrency2>
				<payerPartyReference id="OTHER_BANK_3" />
				<receiverPartyReference id="THIS_BANK" />
				<paymentAmount>
				  <currency>CAD</currency>
				  <amount>1000000</amount>
				</paymentAmount>
			  </exchangedCurrency2>

			  <paymentDate>2021-01-01</paymentDate>
			</fxLeg>

		</Trade>

    </Portfolio>
    """)

# The next step is that we have to build our in-memory structure, we
# can do this by querying the underlying data structure. Note that this
# native Julia syntax will be improved in a later release of DataKnots.

Exchange = 
  IsDict >> 
  Record(:payer => It.payerPartyReference >> IsDict >> It."@id" >> Is(String),
         :receiver => It.receiverPartyReference >> IsDict >> It."@id" >> Is(String),
         :currency => It.paymentAmount >> IsDict >> It.currency >> Is(String),
         :amount => It.paymentAmount >> IsDict >> It.amount >> parse.(Int, It))

Leg =  
  vectorize.(It.fxLeg) >> IsDict >>
  Record(:ex1 => It.exchangedCurrency1 >> Exchange,
         :ex2 => It.exchangedCurrency2 >> Exchange,
         :date => It.paymentDate >> parse.(Date, It)) >>
  Label(:leg)

Header =
  Is1to1(
    It.Header >> IsDict >> 
    Record(:id => It.tradeID >> parse.(Int, It),
           :date => It.tradeDate  >> parse.(Date, It),
           :book => It.book)) >>
  Label(:header)

Trade =
  vectorize.(It.Trade) >> IsDict >>
  Record(Header, Leg,
         :product => It.product >> Is(String)) >>
  Label(:trade)

Portfolio = Record(It.Portfolio >> IsDict >> Trade)

# Now we can load the data, this step doesn't convert the
# data, it simply wraps it as a DataKnot.
d = xml_dict(xml_string)
raw = DataKnot( :Portfolio => d["Portfolio"] )

# If you want, you can directly convert the data; this can be
# expensive since it'll transform the entire input.
knot = raw[Portfolio]

# here is an example query directly against this converted knot 
@query knot {count(trade), trade{header.id, max(leg.ex1.amount)}}
#=>
│ #A  trade{id,#B}                                   │
┼────────────────────────────────────────────────────┼
│  3  10001, 1100000; 10002, 1200000; 10003, 1200000 │
=#

# alternatively, you can directly query the raw data; this has
# the advantage of only converting the data needed...
@query raw $Portfolio{count(trade), trade{header.id, max(leg.ex1.amount)}}
#=>
│ #A  trade{id,#B}                                   │
┼────────────────────────────────────────────────────┼
│  3  10001, 1100000; 10002, 1200000; 10003, 1200000 │
=#