Created
November 29, 2021 14:10
-
-
Save hariroshan/40a70421bbc94b0f117521b4ba0e20b6 to your computer and use it in GitHub Desktop.
Yahoo quotes Protobuf stream processing in elm
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
module Api.Proto.YahooStream exposing (..) | |
import Api.Common exposing (PriceQuote) | |
import Array | |
import Bitwise | |
import Time | |
import Utils | |
type PricingData_QuoteType | |
= PricingData_None -- 0 | |
| PricingData_Altsymbol -- 5 | |
| PricingData_Heartbeat -- 7 | |
| PricingData_Equity -- 8 | |
| PricingData_Index -- 9 | |
| PricingData_Mutualfund -- 11 | |
| PricingData_Moneymarket -- 12 | |
| PricingData_Option -- 13 | |
| PricingData_Currency -- 14 | |
| PricingData_Warrant -- 15 | |
| PricingData_Bond -- 17 | |
| PricingData_Future -- 18 | |
| PricingData_Etf -- 20 | |
| PricingData_Commodity -- 23 | |
| PricingData_Ecnquote -- 28 | |
| PricingData_Cryptocurrency -- 41 | |
| PricingData_Indicator -- 42 | |
| PricingData_Industry -- 1000 | |
type PricingData_OptionType | |
= PricingData_Call -- 0 | |
| PricingData_Put -- 1 | |
type PricingData_MarketHoursType | |
= PricingData_PreMarket -- 0 | |
| PricingData_RegularMarket -- 1 | |
| PricingData_PostMarket -- 2 | |
| PricingData_ExtendedHoursMarket -- 3 | |
type alias PricingData = | |
{ id : String -- 1 | |
, price : Float -- 2 | |
, time : Int -- 3 | |
, currency : String -- 4 | |
, exchange : String -- 5 | |
, quoteType : PricingData_QuoteType -- 6 | |
, marketHours : PricingData_MarketHoursType -- 7 | |
, changePercent : Float -- 8 | |
, dayVolume : Int -- 9 | |
, dayHigh : Float -- 10 | |
, dayLow : Float -- 11 | |
, change : Float -- 12 | |
, shortName : String -- 13 | |
, expireDate : Int -- 14 | |
, openPrice : Float -- 15 | |
, previousClose : Float -- 16 | |
, strikePrice : Float -- 17 | |
, underlyingSymbol : String -- 18 | |
, openInterest : Int -- 19 | |
, optionsType : PricingData_OptionType -- 20 | |
, miniOption : Int -- 21 | |
, lastSize : Int -- 22 | |
, bid : Float -- 23 | |
, bidSize : Int -- 24 | |
, ask : Float -- 25 | |
, askSize : Int -- 26 | |
, priceHint : Int -- 27 | |
, vol24Hr : Int -- 28 | |
, volAllCurrencies : Int -- 29 | |
, fromcurrency : String -- 30 | |
, lastMarket : String -- 31 | |
, circulatingSupply : Float -- 32 | |
, marketcap : Float -- 33 | |
} | |
type alias QuoteRecord = | |
{ id : String | |
, price : Float | |
, time : Int | |
, exchange : String | |
, quoteType : Int | |
, marketHours : Int | |
, changePercent : Float | |
, dayVolume : Int | |
, change : Float | |
, priceHint : Int | |
, currency : String | |
, dayHigh : Float | |
, dayLow : Float | |
, shortName : String | |
, expireDate : Int | |
, openPrice : Float | |
, previousClose : Float | |
, strikePrice : Float | |
, underlyingSymbol : String | |
, openInterest : Int | |
, optionsType : Int | |
, miniOption : Int | |
, lastSize : Int | |
, bid : Float | |
, bidSize : Int | |
, ask : Float | |
, askSize : Int | |
, vol24hr : Int | |
, volAllCurrencies : Int | |
, fromcurrency : String | |
, lastMarket : String | |
, components : String | |
, circulatingSupply : Float | |
, marketcap : Float | |
} | |
chars : String | |
chars = | |
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" | |
defaultPrevValue : number | |
defaultPrevValue = | |
4294967295 | |
parseUInt : Int -> List Int -> ( Int, List Int ) | |
parseUInt prevValue xs = | |
let | |
helper prevVal dep ls = | |
case ls of | |
[] -> | |
( 0, [] ) | |
x :: rest -> | |
let | |
value = | |
Bitwise.or | |
(if dep == 0 then | |
0 | |
else | |
prevVal | |
) | |
(Bitwise.and x | |
(if dep == 4 then | |
15 | |
else | |
127 | |
) | |
|> Bitwise.shiftLeftBy (7 * dep) | |
) | |
|> Bitwise.shiftRightZfBy 0 | |
in | |
if x < 128 then | |
( value, rest ) | |
else | |
helper value (dep + 1) rest | |
in | |
helper prevValue 0 xs | |
readUintLE : List Int -> Int | |
readUintLE xs = | |
case xs of | |
a :: b :: c :: d :: _ -> | |
let | |
bShift = | |
b |> Bitwise.shiftLeftBy 8 | |
cShift = | |
c |> Bitwise.shiftLeftBy 16 | |
dShift = | |
d |> Bitwise.shiftLeftBy 24 | |
in | |
Bitwise.or a bShift | |
|> Bitwise.or cShift | |
|> Bitwise.or dShift | |
|> Bitwise.shiftRightZfBy 0 | |
_ -> | |
0 | |
parseDouble : List Int -> ( Float, List Int ) | |
parseDouble xs = | |
let | |
lo = | |
readUintLE xs | |
hi = | |
readUintLE (xs |> List.drop 4) | |
sign = | |
(hi |> Bitwise.shiftRightBy 31) * 2 + 1 | |
exponent = | |
Bitwise.and (hi |> Bitwise.shiftRightZfBy 20) 2047 | |
mantissa = | |
4294967296 * Bitwise.and hi 1048575 + lo | |
value = | |
if exponent == 2047 then | |
if mantissa /= 0 then | |
-1 | |
else | |
-1 | |
else if exponent == 0 then | |
toFloat sign * 5.0e-324 * toFloat mantissa | |
else | |
toFloat sign * (2 ^ toFloat (exponent - 1075)) * toFloat (mantissa + 4503599627370496) | |
in | |
( value, xs |> List.drop 8 ) | |
parseFloat : List Int -> ( Float, List Int ) | |
parseFloat xs = | |
let | |
uint = | |
readUintLE xs | |
sign = | |
(uint |> Bitwise.shiftRightBy 31) * 2 + 1 | |
exponent = | |
uint |> Bitwise.shiftRightZfBy 23 |> Bitwise.and 255 | |
mantissa = | |
uint |> Bitwise.and 8388607 | |
value = | |
if exponent == 255 then | |
if mantissa /= 0 then | |
-1 | |
else | |
-1 | |
else if exponent == 0 then | |
toFloat sign * 1.401298464324817e-45 * toFloat mantissa | |
else | |
toFloat sign * toFloat (2 ^ (exponent - 150)) * toFloat (mantissa + 8388608) | |
in | |
( value, xs |> List.drop 4 ) | |
readLongVaint : List Int -> { lo : Int, hi : Int, rest : List Int } | |
readLongVaint xs = | |
let | |
len = | |
xs |> List.length | |
in | |
Utils.loop | |
(\{ lo, hi } -> | |
if not (len > 4) then | |
let | |
( btlow, i ) = | |
xs | |
|> Utils.foldlFast | |
(\cur ( acc1, idx ) -> | |
let | |
btLow = | |
Bitwise.or acc1 | |
(cur | |
|> Bitwise.and 127 | |
|> Bitwise.shiftLeftBy (7 * idx) | |
) | |
|> Bitwise.shiftRightZfBy 0 | |
in | |
if cur < 128 then | |
( ( btLow, idx ), Utils.Stop ) | |
else | |
( ( btLow, idx + 1 ), Utils.Continue ) | |
) | |
( 0, 0 ) | |
in | |
( { lo = btlow, hi = hi, rest = List.drop (i + 1) xs }, Utils.Stop ) | |
else | |
let | |
( btlow, consumed ) = | |
List.take 4 xs | |
|> Utils.foldlFast | |
(\cur ( acc1, i ) -> | |
let | |
btLow = | |
Bitwise.or acc1 | |
(cur | |
|> Bitwise.and 127 | |
|> Bitwise.shiftLeftBy (7 * i) | |
) | |
|> Bitwise.shiftRightZfBy 0 | |
in | |
if cur < 128 then | |
( ( btLow, i ), Utils.Stop ) | |
else | |
( ( btLow, i + 1 ), Utils.Continue ) | |
) | |
( lo, 0 ) | |
in | |
if consumed < 4 then | |
( { lo = btlow, hi = hi, rest = List.drop (consumed + 1) xs }, Utils.Stop ) | |
else | |
let | |
rest = | |
List.drop 4 xs | |
in | |
case List.head rest of | |
Nothing -> | |
( { lo = btlow, hi = hi, rest = [] }, Utils.Stop ) | |
Just h -> | |
let | |
nbtLow = | |
Bitwise.or btlow | |
(h | |
|> Bitwise.and 127 | |
|> Bitwise.shiftLeftBy 28 | |
) | |
|> Bitwise.shiftRightZfBy 0 | |
nbtHigh = | |
Bitwise.or hi | |
(h | |
|> Bitwise.and 127 | |
|> Bitwise.shiftRightBy 4 | |
) | |
|> Bitwise.shiftRightZfBy 0 | |
consumedAll = | |
rest |> List.drop 1 | |
in | |
if h < 128 then | |
( { lo = nbtLow, hi = nbtHigh, rest = consumedAll }, Utils.Stop ) | |
else | |
let | |
( hbits, i ) = | |
consumedAll | |
|> Utils.foldlFast | |
(\cur ( acc1, idx ) -> | |
let | |
res = | |
Bitwise.or acc1 | |
(cur | |
|> Bitwise.and 127 | |
|> Bitwise.shiftLeftBy (idx * 7 + 3) | |
) | |
|> Bitwise.shiftRightZfBy 0 | |
in | |
if cur < 128 then | |
( ( res, idx + 1 ), Utils.Stop ) | |
else | |
( ( res, idx + 1 ), Utils.Continue ) | |
) | |
( nbtHigh, 0 ) | |
in | |
( { lo = nbtLow, hi = hbits, rest = List.drop i consumedAll }, Utils.Stop ) | |
) | |
{ lo = 0, hi = 0, rest = xs } | |
zzDecode : { a | lo : Int, hi : Int } -> { a | lo : Int, hi : Int } | |
zzDecode ({ lo, hi } as a) = | |
let | |
mask = | |
-(lo |> Bitwise.and 1) | |
in | |
{ a | |
| lo = Bitwise.or (lo |> Bitwise.shiftRightZfBy 1) (hi |> Bitwise.shiftLeftBy 31) |> Bitwise.xor mask |> Bitwise.shiftRightZfBy 0 | |
, hi = hi |> Bitwise.shiftRightZfBy 1 |> Bitwise.xor mask |> Bitwise.shiftRightZfBy 0 | |
} | |
toNumber : Bool -> { a | lo : Int, hi : Int } -> Int | |
toNumber unsigned { lo, hi } = | |
if not unsigned && (hi |> Bitwise.shiftRightZfBy 31 |> (/=) 0) then | |
let | |
l = | |
(Bitwise.complement lo + 1) |> Bitwise.shiftRightZfBy 0 | |
h = | |
Bitwise.complement hi |> Bitwise.shiftRightZfBy 0 | |
nH = | |
if l == 0 then | |
h + 1 |> Bitwise.shiftRightZfBy 0 | |
else | |
h | |
in | |
-(l + nH * 4294967296) | |
else | |
lo + hi * 4294967296 | |
parseSint64 : List Int -> ( Int, List Int ) | |
parseSint64 xs = | |
let | |
res = | |
xs | |
|> readLongVaint | |
|> zzDecode | |
-- |> Debug.log "Decode" | |
in | |
( res |> toNumber False, res.rest ) | |
parseInt32 : Int -> List Int -> ( Int, List Int ) | |
parseInt32 prevVal xs = | |
parseUInt prevVal xs |> Tuple.mapFirst (Bitwise.or 0) | |
parseString : Int -> List Int -> ( String, List Int ) | |
parseString prevVal xs = | |
let | |
( length, rest ) = | |
parseUInt prevVal xs | |
in | |
rest | |
|> Utils.foldlFast | |
(\cur ( ( acc, returnRest ), idx ) -> | |
if idx < length then | |
( ( ( Char.fromCode cur :: acc, returnRest |> List.drop 1 ), idx + 1 ), Utils.Continue ) | |
else | |
( ( ( acc, returnRest ), idx ), Utils.Stop ) | |
) | |
( ( [], rest ), 0 ) | |
|> Tuple.first | |
|> Tuple.mapFirst (List.reverse >> String.fromList) | |
atob : (Char -> a) -> String -> List a | |
atob transform input = | |
if modBy 4 (String.length input) == 1 then | |
[] | |
else | |
let | |
arr = | |
input | |
|> String.toList | |
|> List.filter ((/=) '=') | |
|> Array.fromList | |
in | |
arr | |
|> Array.foldl | |
(\buffer ({ bc, bs, output } as acc) -> | |
String.indexes (String.fromChar buffer) chars | |
|> List.head | |
|> Maybe.map | |
(\charcode -> | |
let | |
bufferComplement = | |
Bitwise.complement charcode | |
newBs = | |
if modBy 4 bc /= 0 then | |
bs * 64 + charcode | |
else | |
charcode | |
newBc = | |
bc + 1 | |
newOutput = | |
if bufferComplement /= 0 && modBy 4 bc /= 0 then | |
Char.fromCode | |
(Bitwise.and 255 (newBs |> Bitwise.shiftRightBy (-2 * newBc |> Bitwise.and 6))) | |
:: output | |
else | |
output | |
in | |
{ bc = newBc, bs = newBs, output = newOutput } | |
) | |
|> Maybe.withDefault acc | |
) | |
{ bc = 0, bs = 0, output = [] } | |
|> .output | |
|> List.foldl (\cur acc -> transform cur :: acc) [] | |
{- | |
case 35: | |
r.indices && r.indices.length || (r.indices = []), r.indices.push(e.string()); | |
break; | |
default: | |
e.skipType(7 & i) | |
-} | |
updateRecord : Int -> Int -> List Int -> QuoteRecord -> ( List Int, QuoteRecord ) | |
updateRecord i prevVal rest rec = | |
case Bitwise.shiftRightZfBy 3 i of | |
-- |> Debug.log "SHIFT" | |
1 -> | |
let | |
( id, newRest ) = | |
parseString prevVal rest | |
in | |
( newRest, { rec | id = id } ) | |
2 -> | |
let | |
( price, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | price = price } ) | |
3 -> | |
let | |
( time, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | time = time } ) | |
4 -> | |
let | |
( currency, newRest ) = | |
parseString prevVal rest | |
in | |
( newRest, { rec | currency = currency } ) | |
5 -> | |
let | |
( exchange, newRest ) = | |
parseString prevVal rest | |
in | |
( newRest, { rec | exchange = exchange } ) | |
6 -> | |
let | |
( quoteType, newRest ) = | |
parseInt32 prevVal rest | |
in | |
( newRest, { rec | quoteType = quoteType } ) | |
7 -> | |
let | |
( marketHours, newRest ) = | |
parseInt32 prevVal rest | |
in | |
( newRest, { rec | marketHours = marketHours } ) | |
8 -> | |
let | |
( changePercent, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | changePercent = changePercent } ) | |
9 -> | |
let | |
( dayVolume, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | dayVolume = dayVolume } ) | |
10 -> | |
let | |
( dayHigh, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | dayHigh = dayHigh } ) | |
11 -> | |
let | |
( dayLow, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | dayLow = dayLow } ) | |
12 -> | |
let | |
( change, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | change = change } ) | |
13 -> | |
let | |
( shortName, newRest ) = | |
parseString prevVal rest | |
in | |
( newRest, { rec | shortName = shortName } ) | |
14 -> | |
let | |
( expireDate, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | expireDate = expireDate } ) | |
15 -> | |
let | |
( openPrice, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | openPrice = openPrice } ) | |
16 -> | |
let | |
( previousClose, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | previousClose = previousClose } ) | |
17 -> | |
let | |
( strikePrice, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | strikePrice = strikePrice } ) | |
18 -> | |
let | |
( underlyingSymbol, newRest ) = | |
parseString prevVal rest | |
in | |
( newRest, { rec | underlyingSymbol = underlyingSymbol } ) | |
19 -> | |
let | |
( openInterest, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | openInterest = openInterest } ) | |
20 -> | |
let | |
( optionsType, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | optionsType = optionsType } ) | |
21 -> | |
let | |
( miniOption, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | miniOption = miniOption } ) | |
22 -> | |
let | |
( lastSize, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | lastSize = lastSize } ) | |
23 -> | |
let | |
( bid, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | bid = bid } ) | |
24 -> | |
let | |
( bidSize, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | bidSize = bidSize } ) | |
25 -> | |
let | |
( ask, newRest ) = | |
parseFloat rest | |
in | |
( newRest, { rec | ask = ask } ) | |
26 -> | |
let | |
( askSize, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | askSize = askSize } ) | |
27 -> | |
let | |
( priceHint, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | priceHint = priceHint } ) | |
28 -> | |
let | |
( vol24hr, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | vol24hr = vol24hr } ) | |
29 -> | |
let | |
( volAllCurrencies, newRest ) = | |
parseSint64 rest | |
in | |
( newRest, { rec | volAllCurrencies = volAllCurrencies } ) | |
30 -> | |
let | |
( fromcurrency, newRest ) = | |
parseString prevVal rest | |
in | |
( newRest, { rec | fromcurrency = fromcurrency } ) | |
31 -> | |
let | |
( lastMarket, newRest ) = | |
parseString prevVal rest | |
in | |
( newRest, { rec | lastMarket = lastMarket } ) | |
32 -> | |
let | |
( circulatingSupply, newRest ) = | |
parseDouble rest | |
in | |
( newRest, { rec | circulatingSupply = circulatingSupply } ) | |
33 -> | |
let | |
( marketcap, newRest ) = | |
parseDouble rest | |
in | |
( newRest, { rec | marketcap = marketcap } ) | |
34 -> | |
let | |
( components, newRest ) = | |
parseString prevVal rest | |
in | |
( newRest, { rec | components = components } ) | |
_ -> | |
-- Hopefull it will not it | |
( rest |> List.drop 4, rec ) | |
-- e.skipType(7 & i) | |
-- case Bitwise.and i 7 of | |
-- -- 5 -> | |
-- -- ( rest |> List.drop 4, rec ) | |
-- _ -> | |
-- i | |
-- |> Debug.log "AA" | |
-- |> Debug.todo "IMPLE" | |
parseMsg : String -> PriceQuote | |
parseMsg msg = | |
let | |
result = | |
-- "CgReVE5YFTMz0z8YoJnRz6lfKgNOWUIwCTgBRULsxD9lwMzMPNgBCA==" | |
-- "CgVeRlRTRRUKKeNFGMCkpdKpXyoDRkdJMAk4AUU67kA+ZQC4WkHYAQQ=" | |
-- "CgxUQVRBU1RFRUwuTlMVM2OVRBjQjJmLql8qA05TSTAIOAFFgf+CPkjOpoMDZQAyQ0CwAWjYAQQ=" | |
msg | |
|> atob | |
(Char.toCode >> modBy 256 | |
-- 8Bit List | |
) | |
in | |
Utils.loop | |
(\( ( acc, priceData ), ( idx, prevVal ) ) -> | |
let | |
( i, rest ) = | |
parseUInt prevVal acc | |
newPrevVal = | |
i | |
res = | |
updateRecord | |
i | |
newPrevVal | |
rest | |
priceData | |
in | |
( ( res | |
, ( idx + 1, newPrevVal ) | |
) | |
, if res |> Tuple.first |> List.isEmpty then | |
Utils.Stop | |
else | |
Utils.Continue | |
) | |
) | |
( ( result | |
, { id = "" | |
, price = 0.0 | |
, time = 0 | |
, exchange = "" | |
, quoteType = 0 | |
, marketHours = 0 | |
, changePercent = 0.0 | |
, dayVolume = 0 | |
, change = 0.0 | |
, priceHint = 0 | |
, currency = "" | |
, dayHigh = 0.0 | |
, dayLow = 0.0 | |
, shortName = "" | |
, expireDate = 0 | |
, openPrice = 0.0 | |
, previousClose = 0.0 | |
, strikePrice = 0.0 | |
, underlyingSymbol = "" | |
, openInterest = 0 | |
, optionsType = 0 | |
, miniOption = 0 | |
, lastSize = 0 | |
, bid = 0.0 | |
, bidSize = 0 | |
, ask = 0.0 | |
, askSize = 0 | |
, vol24hr = 0 | |
, volAllCurrencies = 0 | |
, fromcurrency = "" | |
, lastMarket = "" | |
, components = "" | |
, circulatingSupply = 0.0 | |
, marketcap = 0.0 | |
} | |
) | |
, ( 0, defaultPrevValue ) | |
) | |
|> Tuple.first | |
|> Tuple.second | |
-- |> Debug.log "RECEIVED" | |
|> toPriceQuote | |
{- | |
type alias PriceQuote = | |
{ symbol : String | |
, exchange : String | |
, price : Float | |
, tickPrecision : Int | |
, time : Time.Posix | |
} | |
-} | |
toPriceQuote : QuoteRecord -> PriceQuote | |
toPriceQuote { id, exchange, price, priceHint, time, dayVolume } = | |
{ symbol = id | |
, exchange = exchange | |
, price = price | |
, tickPrecision = priceHint | |
, time = Time.millisToPosix time | |
, dayVolume = dayVolume | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment