Skip to content

Instantly share code, notes, and snippets.

@hpoit

hpoit/pvt Secret

Created May 4, 2017 17:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save hpoit/cb772dfd860731a46d95bb7955558272 to your computer and use it in GitHub Desktop.
Save hpoit/cb772dfd860731a46d95bb7955558272 to your computer and use it in GitHub Desktop.
Download ZIP
pvt
Raw
pvt
julia> using CSV, DataFrames, Query, Libz
WARNING: Method definition describe(AbstractArray) in module StatsBase at /Users/Corvus/.julia/v0.5/StatsBase/src/scalarstats.jl:573 overwritten in module DataFrames at /Users/Corvus/.julia/v0.5/DataFrames/src/abstractdataframe/abstractdataframe.jl:407.
WARNING: Method definition require(Symbol) in module Base at loading.jl:345 overwritten in module Query at /Users/Corvus/.julia/v0.5/Requires/src/require.jl:12.
INFO: Recompiling stale cache file /Users/Corvus/.julia/lib/v0.5/BufferedStreams.ji for module BufferedStreams.
INFO: Recompiling stale cache file /Users/Corvus/.julia/lib/v0.5/Libz.ji for module Libz.
julia> Base.nb_available{T <: Libz.Source}(s::BufferedStreams.BufferedInputStream{T}) = eof(s)
julia> io = Libz.ZlibInflateInputStream(open("/file.csv.gz", "r"))
BufferedStreams.BufferedInputStream{Libz.Source{:inflate,BufferedStreams.BufferedInputStream{IOStream}}}(<8.0 KiB buffer, 0% filled>)
julia> q = @from i in CSV.Source(io) begin
@where i.Type == "Trade"
@select {i.Price, i.Volume, i.Time_G_}
end
Query.EnumerableConvert2Nullable{NamedTuples._NT_,Query.EnumerableSelect{Union{},Query.EnumerableWhere{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String}},Query.EnumerableConvert2DataValue{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String}},Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}}}}}},##1#3},##2#4}}(Query.EnumerableSelect{Union{},Query.EnumerableWhere{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String}},Query.EnumerableConvert2DataValue{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String}},Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}}}}}},##1#3},##2#4}(Query.EnumerableWhere{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String}},Query.EnumerableConvert2DataValue{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String}},Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}}}}}},##1#3}(Query.EnumerableConvert2DataValue{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{String}},Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}}}}}}(Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}}}}}(IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}}}}(CSV.Source: __IO__
CSV.Options:
delim: ','
quotechar: '"'
escapechar: '\\'
null: ""
dateformat: Base.Dates.DateFormat(Base.Dates.Slot[Base.Dates.DelimitedSlot{Base.Dates.Year}(Base.Dates.Year,'y',4,"-"),Base.Dates.DelimitedSlot{Base.Dates.Month}(Base.Dates.Month,'m',2,"-"),Base.Dates.DelimitedSlot{Base.Dates.Day}(Base.Dates.Day,'d',2,r"(?=\s|$)")],"","english")
Data.Schema{true}:
rows: 0 cols: 12
Columns:
"#RIC" Nullable{WeakRefString{UInt8}}
"Date[G]" Nullable{WeakRefString{UInt8}}
"Time[G]" Nullable{WeakRefString{UInt8}}
"GMT Offset" Nullable{WeakRefString{UInt8}}
"Type" Nullable{WeakRefString{UInt8}}
"Price" Nullable{WeakRefString{UInt8}}
"Volume" Nullable{WeakRefString{UInt8}}
"Bid Price" Nullable{WeakRefString{UInt8}}
"Bid Size" Nullable{WeakRefString{UInt8}}
"Ask Price" Nullable{WeakRefString{UInt8}}
"Ask Size" Nullable{WeakRefString{UInt8}}
"Qualifiers" Nullable{WeakRefString{UInt8}},Data.Schema{true}:
rows: 0 cols: 12
Columns:
"#RIC" Nullable{WeakRefString{UInt8}}
"Date[G]" Nullable{WeakRefString{UInt8}}
"Time[G]" Nullable{WeakRefString{UInt8}}
"GMT Offset" Nullable{WeakRefString{UInt8}}
"Type" Nullable{WeakRefString{UInt8}}
"Price" Nullable{WeakRefString{UInt8}}
"Volume" Nullable{WeakRefString{UInt8}}
"Bid Price" Nullable{WeakRefString{UInt8}}
"Bid Size" Nullable{WeakRefString{UInt8}}
"Ask Price" Nullable{WeakRefString{UInt8}}
"Ask Size" Nullable{WeakRefString{UInt8}}
"Qualifiers" Nullable{WeakRefString{UInt8}}))),#1),#2))
julia> df = DataFrame(take(q, 10_000_000))
0×0 DataFrames.DataFrame
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment