Skip to content

Instantly share code, notes, and snippets.

@hpoit

hpoit/pvt2 with @time and seekstart Secret

Last active May 5, 2017 20:17
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save hpoit/249d3d630c17563c2bc4fcd4be01d6f4 to your computer and use it in GitHub Desktop.
Save hpoit/249d3d630c17563c2bc4fcd4be01d6f4 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
pvt2 with @time and seekstart
Last login: Fri May 5 17:08:19 on ttys000
_
_ _ _(_)_ | A fresh approach to technical computing
(_) | (_) (_) | Documentation: http://docs.julialang.org
_ _ _| |_ __ _ | Type "?help" for help.
| | | | | | |/ _` | |
| | |_| | | | (_| | | Version 0.5.0 (2016-09-19 18:14 UTC)
_/ |\__'_|_|_|\__'_| | Official http://julialang.org/ release
|__/ | x86_64-apple-darwin13.4.0
julia> using CSV, DataFrames, Query, Libz
WARNING: Method definition describe(AbstractArray) in module StatsBase at /Users/Corvus/.julia/v0.5/StatsBase/src/scalarstats.jl:573 overwritten in module DataFrames at /Users/Corvus/.julia/v0.5/DataFrames/src/abstractdataframe/abstractdataframe.jl:407.
WARNING: Method definition require(Symbol) in module Base at loading.jl:345 overwritten in module Query at /Users/Corvus/.julia/v0.5/Requires/src/require.jl:12.
julia> if !method_exists(nb_available, (BufferedStreams.BufferedInputStream{Libz.Source}, ))
Base.nb_available{T <: Libz.Source}(s::BufferedStreams.BufferedInputStream{T}) = !eof(s)
end
julia> io = Libz.ZlibInflateInputStream(open("file.csv.gz", "r"))
BufferedStreams.BufferedInputStream{Libz.Source{:inflate,BufferedStreams.BufferedInputStream{IOStream}}}(<8.0 KiB buffer, 0% filled>)
julia> function query()
q = @from i in CSV.Source(io) begin
@where i.Type == "Trade"
@select {i.Price, i.Volume, i.Time_G_} # pvt2
end
end
query (generic function with 1 method)
julia> @time query()
245.392852 seconds (5.28 M allocations: 8.289 GB, 1.01% gc time)
Query.EnumerableConvert2Nullable{NamedTuples._NT_,Query.EnumerableSelect{Union{},Query.EnumerableWhere{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{Int64},Query.DataValue{String},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{String}},Query.EnumerableConvert2DataValue{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{Int64},Query.DataValue{String},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{String}},Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{Int64},Nullable{WeakRefString{UInt8}},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{WeakRefString{UInt8}}}}}},##1#3},##2#4}}(Query.EnumerableSelect{Union{},Query.EnumerableWhere{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{Int64},Query.DataValue{String},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{String}},Query.EnumerableConvert2DataValue{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{Int64},Query.DataValue{String},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{String}},Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{Int64},Nullable{WeakRefString{UInt8}},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{WeakRefString{UInt8}}}}}},##1#3},##2#4}(Query.EnumerableWhere{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{Int64},Query.DataValue{String},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{String}},Query.EnumerableConvert2DataValue{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{Int64},Query.DataValue{String},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{String}},Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{Int64},Nullable{WeakRefString{UInt8}},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{WeakRefString{UInt8}}}}}},##1#3}(Query.EnumerableConvert2DataValue{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Query.DataValue{String},Query.DataValue{String},Query.DataValue{String},Query.DataValue{Int64},Query.DataValue{String},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{Float64},Query.DataValue{Int64},Query.DataValue{String}},Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{Int64},Nullable{WeakRefString{UInt8}},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{WeakRefString{UInt8}}}}}}(Query.EnumerableIterable{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{Int64},Nullable{WeakRefString{UInt8}},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{WeakRefString{UInt8}}}}}(IterableTables.DataStreamIterator{NamedTuples._NT_#RIC_Date[G]_Time[G]_GMT Offset_Type_Price_Volume_Bid Price_Bid Size_Ask Price_Ask Size_Qualifiers{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},CSV.Source,Tuple{Nullable{String},Nullable{String},Nullable{String},Nullable{Int64},Nullable{String},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{String}},Tuple{Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{WeakRefString{UInt8}},Nullable{Int64},Nullable{WeakRefString{UInt8}},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{Float64},Nullable{Int64},Nullable{WeakRefString{UInt8}}}}(CSV.Source: __IO__
CSV.Options:
delim: ','
quotechar: '"'
escapechar: '\\'
null: ""
dateformat: Base.Dates.DateFormat(Base.Dates.Slot[Base.Dates.DelimitedSlot{Base.Dates.Year}(Base.Dates.Year,'y',4,"-"),Base.Dates.DelimitedSlot{Base.Dates.Month}(Base.Dates.Month,'m',2,"-"),Base.Dates.DelimitedSlot{Base.Dates.Day}(Base.Dates.Day,'d',2,r"(?=\s|$)")],"","english")
Data.Schema{true}:
rows: 87518320 cols: 12
Columns:
"#RIC" Nullable{WeakRefString{UInt8}}
"Date[G]" Nullable{WeakRefString{UInt8}}
"Time[G]" Nullable{WeakRefString{UInt8}}
"GMT Offset" Nullable{Int64}
"Type" Nullable{WeakRefString{UInt8}}
"Price" Nullable{Float64}
"Volume" Nullable{Int64}
"Bid Price" Nullable{Float64}
"Bid Size" Nullable{Int64}
"Ask Price" Nullable{Float64}
"Ask Size" Nullable{Int64}
"Qualifiers" Nullable{WeakRefString{UInt8}},Data.Schema{true}:
rows: 87518320 cols: 12
Columns:
"#RIC" Nullable{WeakRefString{UInt8}}
"Date[G]" Nullable{WeakRefString{UInt8}}
"Time[G]" Nullable{WeakRefString{UInt8}}
"GMT Offset" Nullable{Int64}
"Type" Nullable{WeakRefString{UInt8}}
"Price" Nullable{Float64}
"Volume" Nullable{Int64}
"Bid Price" Nullable{Float64}
"Bid Size" Nullable{Int64}
"Ask Price" Nullable{Float64}
"Ask Size" Nullable{Int64}
"Qualifiers" Nullable{WeakRefString{UInt8}}))),#1),#2))
julia> seekstart(io)
ERROR: ArgumentError: Can't seek in input stream with source of type Libz.Source{:inflate,BufferedStreams.BufferedInputStream{IOStream}}
in seek(::BufferedStreams.BufferedInputStream{Libz.Source{:inflate,BufferedStreams.BufferedInputStream{IOStream}}}, ::Int64) at /Users/Corvus/.julia/v0.5/BufferedStreams/src/bufferedinputstream.jl:323
in seekstart(::BufferedStreams.BufferedInputStream{Libz.Source{:inflate,BufferedStreams.BufferedInputStream{IOStream}}}) at ./iostream.jl:54
julia>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment