Skip to content

Instantly share code, notes, and snippets.

@hariroshan
Created November 29, 2021 14:10
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save hariroshan/40a70421bbc94b0f117521b4ba0e20b6 to your computer and use it in GitHub Desktop.
Save hariroshan/40a70421bbc94b0f117521b4ba0e20b6 to your computer and use it in GitHub Desktop.
Yahoo quotes Protobuf stream processing in elm
module Api.Proto.YahooStream exposing (..)
import Api.Common exposing (PriceQuote)
import Array
import Bitwise
import Time
import Utils
type PricingData_QuoteType
= PricingData_None -- 0
| PricingData_Altsymbol -- 5
| PricingData_Heartbeat -- 7
| PricingData_Equity -- 8
| PricingData_Index -- 9
| PricingData_Mutualfund -- 11
| PricingData_Moneymarket -- 12
| PricingData_Option -- 13
| PricingData_Currency -- 14
| PricingData_Warrant -- 15
| PricingData_Bond -- 17
| PricingData_Future -- 18
| PricingData_Etf -- 20
| PricingData_Commodity -- 23
| PricingData_Ecnquote -- 28
| PricingData_Cryptocurrency -- 41
| PricingData_Indicator -- 42
| PricingData_Industry -- 1000
type PricingData_OptionType
= PricingData_Call -- 0
| PricingData_Put -- 1
type PricingData_MarketHoursType
= PricingData_PreMarket -- 0
| PricingData_RegularMarket -- 1
| PricingData_PostMarket -- 2
| PricingData_ExtendedHoursMarket -- 3
type alias PricingData =
{ id : String -- 1
, price : Float -- 2
, time : Int -- 3
, currency : String -- 4
, exchange : String -- 5
, quoteType : PricingData_QuoteType -- 6
, marketHours : PricingData_MarketHoursType -- 7
, changePercent : Float -- 8
, dayVolume : Int -- 9
, dayHigh : Float -- 10
, dayLow : Float -- 11
, change : Float -- 12
, shortName : String -- 13
, expireDate : Int -- 14
, openPrice : Float -- 15
, previousClose : Float -- 16
, strikePrice : Float -- 17
, underlyingSymbol : String -- 18
, openInterest : Int -- 19
, optionsType : PricingData_OptionType -- 20
, miniOption : Int -- 21
, lastSize : Int -- 22
, bid : Float -- 23
, bidSize : Int -- 24
, ask : Float -- 25
, askSize : Int -- 26
, priceHint : Int -- 27
, vol24Hr : Int -- 28
, volAllCurrencies : Int -- 29
, fromcurrency : String -- 30
, lastMarket : String -- 31
, circulatingSupply : Float -- 32
, marketcap : Float -- 33
}
type alias QuoteRecord =
{ id : String
, price : Float
, time : Int
, exchange : String
, quoteType : Int
, marketHours : Int
, changePercent : Float
, dayVolume : Int
, change : Float
, priceHint : Int
, currency : String
, dayHigh : Float
, dayLow : Float
, shortName : String
, expireDate : Int
, openPrice : Float
, previousClose : Float
, strikePrice : Float
, underlyingSymbol : String
, openInterest : Int
, optionsType : Int
, miniOption : Int
, lastSize : Int
, bid : Float
, bidSize : Int
, ask : Float
, askSize : Int
, vol24hr : Int
, volAllCurrencies : Int
, fromcurrency : String
, lastMarket : String
, components : String
, circulatingSupply : Float
, marketcap : Float
}
chars : String
chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
defaultPrevValue : number
defaultPrevValue =
4294967295
parseUInt : Int -> List Int -> ( Int, List Int )
parseUInt prevValue xs =
let
helper prevVal dep ls =
case ls of
[] ->
( 0, [] )
x :: rest ->
let
value =
Bitwise.or
(if dep == 0 then
0
else
prevVal
)
(Bitwise.and x
(if dep == 4 then
15
else
127
)
|> Bitwise.shiftLeftBy (7 * dep)
)
|> Bitwise.shiftRightZfBy 0
in
if x < 128 then
( value, rest )
else
helper value (dep + 1) rest
in
helper prevValue 0 xs
readUintLE : List Int -> Int
readUintLE xs =
case xs of
a :: b :: c :: d :: _ ->
let
bShift =
b |> Bitwise.shiftLeftBy 8
cShift =
c |> Bitwise.shiftLeftBy 16
dShift =
d |> Bitwise.shiftLeftBy 24
in
Bitwise.or a bShift
|> Bitwise.or cShift
|> Bitwise.or dShift
|> Bitwise.shiftRightZfBy 0
_ ->
0
parseDouble : List Int -> ( Float, List Int )
parseDouble xs =
let
lo =
readUintLE xs
hi =
readUintLE (xs |> List.drop 4)
sign =
(hi |> Bitwise.shiftRightBy 31) * 2 + 1
exponent =
Bitwise.and (hi |> Bitwise.shiftRightZfBy 20) 2047
mantissa =
4294967296 * Bitwise.and hi 1048575 + lo
value =
if exponent == 2047 then
if mantissa /= 0 then
-1
else
-1
else if exponent == 0 then
toFloat sign * 5.0e-324 * toFloat mantissa
else
toFloat sign * (2 ^ toFloat (exponent - 1075)) * toFloat (mantissa + 4503599627370496)
in
( value, xs |> List.drop 8 )
parseFloat : List Int -> ( Float, List Int )
parseFloat xs =
let
uint =
readUintLE xs
sign =
(uint |> Bitwise.shiftRightBy 31) * 2 + 1
exponent =
uint |> Bitwise.shiftRightZfBy 23 |> Bitwise.and 255
mantissa =
uint |> Bitwise.and 8388607
value =
if exponent == 255 then
if mantissa /= 0 then
-1
else
-1
else if exponent == 0 then
toFloat sign * 1.401298464324817e-45 * toFloat mantissa
else
toFloat sign * toFloat (2 ^ (exponent - 150)) * toFloat (mantissa + 8388608)
in
( value, xs |> List.drop 4 )
readLongVaint : List Int -> { lo : Int, hi : Int, rest : List Int }
readLongVaint xs =
let
len =
xs |> List.length
in
Utils.loop
(\{ lo, hi } ->
if not (len > 4) then
let
( btlow, i ) =
xs
|> Utils.foldlFast
(\cur ( acc1, idx ) ->
let
btLow =
Bitwise.or acc1
(cur
|> Bitwise.and 127
|> Bitwise.shiftLeftBy (7 * idx)
)
|> Bitwise.shiftRightZfBy 0
in
if cur < 128 then
( ( btLow, idx ), Utils.Stop )
else
( ( btLow, idx + 1 ), Utils.Continue )
)
( 0, 0 )
in
( { lo = btlow, hi = hi, rest = List.drop (i + 1) xs }, Utils.Stop )
else
let
( btlow, consumed ) =
List.take 4 xs
|> Utils.foldlFast
(\cur ( acc1, i ) ->
let
btLow =
Bitwise.or acc1
(cur
|> Bitwise.and 127
|> Bitwise.shiftLeftBy (7 * i)
)
|> Bitwise.shiftRightZfBy 0
in
if cur < 128 then
( ( btLow, i ), Utils.Stop )
else
( ( btLow, i + 1 ), Utils.Continue )
)
( lo, 0 )
in
if consumed < 4 then
( { lo = btlow, hi = hi, rest = List.drop (consumed + 1) xs }, Utils.Stop )
else
let
rest =
List.drop 4 xs
in
case List.head rest of
Nothing ->
( { lo = btlow, hi = hi, rest = [] }, Utils.Stop )
Just h ->
let
nbtLow =
Bitwise.or btlow
(h
|> Bitwise.and 127
|> Bitwise.shiftLeftBy 28
)
|> Bitwise.shiftRightZfBy 0
nbtHigh =
Bitwise.or hi
(h
|> Bitwise.and 127
|> Bitwise.shiftRightBy 4
)
|> Bitwise.shiftRightZfBy 0
consumedAll =
rest |> List.drop 1
in
if h < 128 then
( { lo = nbtLow, hi = nbtHigh, rest = consumedAll }, Utils.Stop )
else
let
( hbits, i ) =
consumedAll
|> Utils.foldlFast
(\cur ( acc1, idx ) ->
let
res =
Bitwise.or acc1
(cur
|> Bitwise.and 127
|> Bitwise.shiftLeftBy (idx * 7 + 3)
)
|> Bitwise.shiftRightZfBy 0
in
if cur < 128 then
( ( res, idx + 1 ), Utils.Stop )
else
( ( res, idx + 1 ), Utils.Continue )
)
( nbtHigh, 0 )
in
( { lo = nbtLow, hi = hbits, rest = List.drop i consumedAll }, Utils.Stop )
)
{ lo = 0, hi = 0, rest = xs }
zzDecode : { a | lo : Int, hi : Int } -> { a | lo : Int, hi : Int }
zzDecode ({ lo, hi } as a) =
let
mask =
-(lo |> Bitwise.and 1)
in
{ a
| lo = Bitwise.or (lo |> Bitwise.shiftRightZfBy 1) (hi |> Bitwise.shiftLeftBy 31) |> Bitwise.xor mask |> Bitwise.shiftRightZfBy 0
, hi = hi |> Bitwise.shiftRightZfBy 1 |> Bitwise.xor mask |> Bitwise.shiftRightZfBy 0
}
toNumber : Bool -> { a | lo : Int, hi : Int } -> Int
toNumber unsigned { lo, hi } =
if not unsigned && (hi |> Bitwise.shiftRightZfBy 31 |> (/=) 0) then
let
l =
(Bitwise.complement lo + 1) |> Bitwise.shiftRightZfBy 0
h =
Bitwise.complement hi |> Bitwise.shiftRightZfBy 0
nH =
if l == 0 then
h + 1 |> Bitwise.shiftRightZfBy 0
else
h
in
-(l + nH * 4294967296)
else
lo + hi * 4294967296
parseSint64 : List Int -> ( Int, List Int )
parseSint64 xs =
let
res =
xs
|> readLongVaint
|> zzDecode
-- |> Debug.log "Decode"
in
( res |> toNumber False, res.rest )
parseInt32 : Int -> List Int -> ( Int, List Int )
parseInt32 prevVal xs =
parseUInt prevVal xs |> Tuple.mapFirst (Bitwise.or 0)
parseString : Int -> List Int -> ( String, List Int )
parseString prevVal xs =
let
( length, rest ) =
parseUInt prevVal xs
in
rest
|> Utils.foldlFast
(\cur ( ( acc, returnRest ), idx ) ->
if idx < length then
( ( ( Char.fromCode cur :: acc, returnRest |> List.drop 1 ), idx + 1 ), Utils.Continue )
else
( ( ( acc, returnRest ), idx ), Utils.Stop )
)
( ( [], rest ), 0 )
|> Tuple.first
|> Tuple.mapFirst (List.reverse >> String.fromList)
atob : (Char -> a) -> String -> List a
atob transform input =
if modBy 4 (String.length input) == 1 then
[]
else
let
arr =
input
|> String.toList
|> List.filter ((/=) '=')
|> Array.fromList
in
arr
|> Array.foldl
(\buffer ({ bc, bs, output } as acc) ->
String.indexes (String.fromChar buffer) chars
|> List.head
|> Maybe.map
(\charcode ->
let
bufferComplement =
Bitwise.complement charcode
newBs =
if modBy 4 bc /= 0 then
bs * 64 + charcode
else
charcode
newBc =
bc + 1
newOutput =
if bufferComplement /= 0 && modBy 4 bc /= 0 then
Char.fromCode
(Bitwise.and 255 (newBs |> Bitwise.shiftRightBy (-2 * newBc |> Bitwise.and 6)))
:: output
else
output
in
{ bc = newBc, bs = newBs, output = newOutput }
)
|> Maybe.withDefault acc
)
{ bc = 0, bs = 0, output = [] }
|> .output
|> List.foldl (\cur acc -> transform cur :: acc) []
{-
case 35:
r.indices && r.indices.length || (r.indices = []), r.indices.push(e.string());
break;
default:
e.skipType(7 & i)
-}
updateRecord : Int -> Int -> List Int -> QuoteRecord -> ( List Int, QuoteRecord )
updateRecord i prevVal rest rec =
case Bitwise.shiftRightZfBy 3 i of
-- |> Debug.log "SHIFT"
1 ->
let
( id, newRest ) =
parseString prevVal rest
in
( newRest, { rec | id = id } )
2 ->
let
( price, newRest ) =
parseFloat rest
in
( newRest, { rec | price = price } )
3 ->
let
( time, newRest ) =
parseSint64 rest
in
( newRest, { rec | time = time } )
4 ->
let
( currency, newRest ) =
parseString prevVal rest
in
( newRest, { rec | currency = currency } )
5 ->
let
( exchange, newRest ) =
parseString prevVal rest
in
( newRest, { rec | exchange = exchange } )
6 ->
let
( quoteType, newRest ) =
parseInt32 prevVal rest
in
( newRest, { rec | quoteType = quoteType } )
7 ->
let
( marketHours, newRest ) =
parseInt32 prevVal rest
in
( newRest, { rec | marketHours = marketHours } )
8 ->
let
( changePercent, newRest ) =
parseFloat rest
in
( newRest, { rec | changePercent = changePercent } )
9 ->
let
( dayVolume, newRest ) =
parseSint64 rest
in
( newRest, { rec | dayVolume = dayVolume } )
10 ->
let
( dayHigh, newRest ) =
parseFloat rest
in
( newRest, { rec | dayHigh = dayHigh } )
11 ->
let
( dayLow, newRest ) =
parseFloat rest
in
( newRest, { rec | dayLow = dayLow } )
12 ->
let
( change, newRest ) =
parseFloat rest
in
( newRest, { rec | change = change } )
13 ->
let
( shortName, newRest ) =
parseString prevVal rest
in
( newRest, { rec | shortName = shortName } )
14 ->
let
( expireDate, newRest ) =
parseSint64 rest
in
( newRest, { rec | expireDate = expireDate } )
15 ->
let
( openPrice, newRest ) =
parseFloat rest
in
( newRest, { rec | openPrice = openPrice } )
16 ->
let
( previousClose, newRest ) =
parseFloat rest
in
( newRest, { rec | previousClose = previousClose } )
17 ->
let
( strikePrice, newRest ) =
parseFloat rest
in
( newRest, { rec | strikePrice = strikePrice } )
18 ->
let
( underlyingSymbol, newRest ) =
parseString prevVal rest
in
( newRest, { rec | underlyingSymbol = underlyingSymbol } )
19 ->
let
( openInterest, newRest ) =
parseSint64 rest
in
( newRest, { rec | openInterest = openInterest } )
20 ->
let
( optionsType, newRest ) =
parseSint64 rest
in
( newRest, { rec | optionsType = optionsType } )
21 ->
let
( miniOption, newRest ) =
parseSint64 rest
in
( newRest, { rec | miniOption = miniOption } )
22 ->
let
( lastSize, newRest ) =
parseSint64 rest
in
( newRest, { rec | lastSize = lastSize } )
23 ->
let
( bid, newRest ) =
parseFloat rest
in
( newRest, { rec | bid = bid } )
24 ->
let
( bidSize, newRest ) =
parseSint64 rest
in
( newRest, { rec | bidSize = bidSize } )
25 ->
let
( ask, newRest ) =
parseFloat rest
in
( newRest, { rec | ask = ask } )
26 ->
let
( askSize, newRest ) =
parseSint64 rest
in
( newRest, { rec | askSize = askSize } )
27 ->
let
( priceHint, newRest ) =
parseSint64 rest
in
( newRest, { rec | priceHint = priceHint } )
28 ->
let
( vol24hr, newRest ) =
parseSint64 rest
in
( newRest, { rec | vol24hr = vol24hr } )
29 ->
let
( volAllCurrencies, newRest ) =
parseSint64 rest
in
( newRest, { rec | volAllCurrencies = volAllCurrencies } )
30 ->
let
( fromcurrency, newRest ) =
parseString prevVal rest
in
( newRest, { rec | fromcurrency = fromcurrency } )
31 ->
let
( lastMarket, newRest ) =
parseString prevVal rest
in
( newRest, { rec | lastMarket = lastMarket } )
32 ->
let
( circulatingSupply, newRest ) =
parseDouble rest
in
( newRest, { rec | circulatingSupply = circulatingSupply } )
33 ->
let
( marketcap, newRest ) =
parseDouble rest
in
( newRest, { rec | marketcap = marketcap } )
34 ->
let
( components, newRest ) =
parseString prevVal rest
in
( newRest, { rec | components = components } )
_ ->
-- Hopefull it will not it
( rest |> List.drop 4, rec )
-- e.skipType(7 & i)
-- case Bitwise.and i 7 of
-- -- 5 ->
-- -- ( rest |> List.drop 4, rec )
-- _ ->
-- i
-- |> Debug.log "AA"
-- |> Debug.todo "IMPLE"
parseMsg : String -> PriceQuote
parseMsg msg =
let
result =
-- "CgReVE5YFTMz0z8YoJnRz6lfKgNOWUIwCTgBRULsxD9lwMzMPNgBCA=="
-- "CgVeRlRTRRUKKeNFGMCkpdKpXyoDRkdJMAk4AUU67kA+ZQC4WkHYAQQ="
-- "CgxUQVRBU1RFRUwuTlMVM2OVRBjQjJmLql8qA05TSTAIOAFFgf+CPkjOpoMDZQAyQ0CwAWjYAQQ="
msg
|> atob
(Char.toCode >> modBy 256
-- 8Bit List
)
in
Utils.loop
(\( ( acc, priceData ), ( idx, prevVal ) ) ->
let
( i, rest ) =
parseUInt prevVal acc
newPrevVal =
i
res =
updateRecord
i
newPrevVal
rest
priceData
in
( ( res
, ( idx + 1, newPrevVal )
)
, if res |> Tuple.first |> List.isEmpty then
Utils.Stop
else
Utils.Continue
)
)
( ( result
, { id = ""
, price = 0.0
, time = 0
, exchange = ""
, quoteType = 0
, marketHours = 0
, changePercent = 0.0
, dayVolume = 0
, change = 0.0
, priceHint = 0
, currency = ""
, dayHigh = 0.0
, dayLow = 0.0
, shortName = ""
, expireDate = 0
, openPrice = 0.0
, previousClose = 0.0
, strikePrice = 0.0
, underlyingSymbol = ""
, openInterest = 0
, optionsType = 0
, miniOption = 0
, lastSize = 0
, bid = 0.0
, bidSize = 0
, ask = 0.0
, askSize = 0
, vol24hr = 0
, volAllCurrencies = 0
, fromcurrency = ""
, lastMarket = ""
, components = ""
, circulatingSupply = 0.0
, marketcap = 0.0
}
)
, ( 0, defaultPrevValue )
)
|> Tuple.first
|> Tuple.second
-- |> Debug.log "RECEIVED"
|> toPriceQuote
{-
type alias PriceQuote =
{ symbol : String
, exchange : String
, price : Float
, tickPrecision : Int
, time : Time.Posix
}
-}
toPriceQuote : QuoteRecord -> PriceQuote
toPriceQuote { id, exchange, price, priceHint, time, dayVolume } =
{ symbol = id
, exchange = exchange
, price = price
, tickPrecision = priceHint
, time = Time.millisToPosix time
, dayVolume = dayVolume
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment