Skip to content

Instantly share code, notes, and snippets.

@flare9x

flare9x/1min_data.jl

Last active June 30, 2018 11:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save flare9x/7fc984b6e221ef40ce4b952d81cc2bc6 to your computer and use it in GitHub Desktop.
Save flare9x/7fc984b6e221ef40ce4b952d81cc2bc6 to your computer and use it in GitHub Desktop.
Download ZIP
build any time resolution using 1 minute data
Raw
1min_data.jl
using DataFrames
using CSV
using Missings
# Load 1 minute data - close data sampled every 1 minute
main_data1_1min = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/1.min.reg.sess.es_0830_to_1500.txt",types=[String; String; fill(Float64, 4); Int;Int],header=true)
main_data2_1min = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/1.min.reg.sess.es_0830_to_1500.txt", header=true)
main_data3_1min = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/1.min.reg.sess.es_0830_to_1500.txt", header=true)
test_one = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/10.min.reg.sess.es.txt", header=true)
test = convert(Array{Float64},test_one[:Close])
test_date = convert(Array{String},test_one[:Date])
test_time = convert(Array{String},test_one[:Time])
# Make Date_Time column
# Data 1
a = main_data1_1min[:Date]
b = main_data1_1min[:Time]
out = String[]
temp = String[]
for i in 1:length(a)
temp = map(join,zip([a[i]],[b[i]]), " ")
append!(out,temp)
end
main_data1_1min[:Date_Time] = out
String.(out)
# Convert to date time
main_data1_1min[:Date_Time] = DateTime.(main_data1_1min[:Date_Time],Dates.DateFormat("mm/dd/yyyy H:M"))
# Data 2
a = main_data2_1min[:Date]
b = main_data2_1min[:Time]
out = String[]
temp = String[]
for i in 1:length(a)
temp = map(join,zip([a[i]],[b[i]]), " ")
append!(out,temp)
end
main_data2_1min[:Date_Time] = out
# Convert to date time
main_data2_1min[:Date_Time] = DateTime.(main_data2_1min[:Date_Time],Dates.DateFormat("mm/dd/yyyy H:M"))
# Data 3
a = main_data3_1min[:Date]
b = main_data3_1min[:Time]
out = String[]
temp = String[]
for i in 1:length(a)
temp = map(join,zip([a[i]],[b[i]]), " ")
append!(out,temp)
end
main_data3_1min[:Date_Time] = out
# Convert to date time
main_data3_1min[:Date_Time] = DateTime.(main_data3_1min[:Date_Time],Dates.DateFormat("mm/dd/yyyy H:M"))
# Create index and left join to known 1 minute index
# Index Data 1
data1_time_index = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/time_index/es_reg_sess_time_index.txt", header=true)
data1_time_index = convert(Array{String},data1_time_index[:header])
data1_time_index_1 = data1_time_index
data1_date_index = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/1.day.reg.sess.es.txt", header=true) ####////////////////////////////// Replace with es_D
data1_date_index = convert(Array{String},data1_date_index[:Date])
len = length(data1_date_index)
data1_date_index = repeat(data1_date_index, inner = 390)
# Repeat times
data1_time_index = repeat(data1_time_index, inner=1, outer=len)
# Join Date Time
out = String[]
temp = String[]
for i = 1:length(data1_time_index)
temp = map(join,zip([data1_date_index[i]],[data1_time_index[i]]), " ")
append!(out,temp)
end
data1_date_time_index = out
# Convert to date time format
data1_date_time_index = DateTime.(data1_date_time_index,Dates.DateFormat("mm/dd/yyyy H:M"))
# Make df to prepare to inner join
index_dim = hcat(data1_date_time_index)
data1_date_time_index = DataFrame(index_dim)
data1_date_time_index = hcat(data1_date_time_index,data1_date_index,data1_time_index)
colnames = ["Date_Time","Date_index","Time_index"]
names!(data1_date_time_index.colindex, map(parse, colnames))
# Data 1 Data Frame
main_data1_1min = join(data1_date_time_index,main_data1_1min, on= :Date_Time, kind = :left,makeunique=true)
# Index Data 2
data2_time_index = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/time_index/es_reg_sess_time_index.txt", header=true)
data2_time_index = convert(Array{String},data2_time_index[:header])
data2_time_index_1 = data2_time_index
data2_date_index = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/1.day.reg.sess.es.txt", header=true) ####////////////////////////////// Replace with es_D
data2_date_index = convert(Array{String},data2_date_index[:Date])
len = length(data2_date_index)
data2_date_index = repeat(data2_date_index, inner = 390) # Es has 390 1 minute bars from 08:31 to 15:00
# Repeat times
data2_time_index = repeat(data2_time_index, inner=1, outer=len)
# Join Date Time
out = String[]
temp = String[]
for i = 1:length(data2_time_index)
temp = map(join,zip([data2_date_index[i]],[data2_time_index[i]]), " ")
append!(out,temp)
end
data2_date_time_index = out
# Convert to date time format
data2_date_time_index = DateTime.(data2_date_time_index,Dates.DateFormat("mm/dd/yyyy H:M"))
# Make df to prepare to inner join
index_dim = hcat(data2_date_time_index)
data2_date_time_index = DataFrame(index_dim)
data2_date_time_index = hcat(data2_date_time_index,data2_date_index,data2_time_index)
colnames = ["Date_Time","Date_index","Time_index"]
names!(data2_date_time_index.colindex, map(parse, colnames))
# Data 2 Data Frame
main_data2_1min = join(data2_date_time_index,main_data2_1min, on= :Date_Time, kind = :left,makeunique=true)
# Index Data 3
data3_time_index = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/time_index/es_reg_sess_time_index.txt", header=true)
data3_time_index = convert(Array{String},data3_time_index[:header])
data3_time_index_1 = data3_time_index
data3_date_index = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/1.day.reg.sess.es.txt", header=true) ####////////////////////////////// Replace with es_D
data3_date_index = convert(Array{String},data3_date_index[:Date])
len = length(data3_date_index)
data3_date_index = repeat(data3_date_index, inner = 390) # Es has 390 1 minute bars from 08:31 to 15:00
# Repeat times
data3_time_index = repeat(data3_time_index, inner=1, outer=len)
# Join Date Time
out = String[]
temp = String[]
for i = 1:length(data3_time_index)
temp = map(join,zip([data3_date_index[i]],[data3_time_index[i]]), " ")
append!(out,temp)
end
data3_date_time_index = out
# Convert to date time format
data3_date_time_index = DateTime.(data3_date_time_index,Dates.DateFormat("mm/dd/yyyy H:M"))
# Make df to prepare to inner join
index_dim = hcat(data3_date_time_index)
data3_date_time_index = DataFrame(index_dim)
data3_date_time_index = hcat(data3_date_time_index,data3_date_index,data3_time_index)
colnames = ["Date_Time","Date_index","Time_index"]
names!(data3_date_time_index.colindex, map(parse, colnames))
# Data 3 Data Frame
main_data3_1min = join(data3_date_time_index,main_data3_1min, on= :Date_Time, kind = :left,makeunique=true)
# Build arrays
# Data 1
data1_date_time_1min_i = main_data1_1min[:Date_Time]
data1_date_1min_i = String.(collect(Missings.replace(main_data1_1min[:Date_index], "")))
data1_time_1min_i = String.(collect(Missings.replace(main_data1_1min[:Time_index], "")))
data1_open_1min_i = Float64.(collect(Missings.replace(main_data1_1min[:Open], 0.0)))
data1_high_1min_i = Float64.(collect(Missings.replace(main_data1_1min[:High], 0.0)))
data1_low_1min_i = Float64.(collect(Missings.replace(main_data1_1min[:Low], 0.0)))
data1_close_1min_i = Float64.(collect(Missings.replace(main_data1_1min[:Close], 0.0)))
# Data 2
data2_date_time_1min_i = main_data2_1min[:Date_Time]
data2_date_1min_i = String.(collect(Missings.replace(main_data2_1min[:Date_index], "")))
data2_time_1min_i = String.(collect(Missings.replace(main_data2_1min[:Time_index], "")))
data2_open_1min_i = Float64.(collect(Missings.replace(main_data2_1min[:Open], 0.0)))
data2_high_1min_i = Float64.(collect(Missings.replace(main_data2_1min[:High], 0.0)))
data2_low_1min_i = Float64.(collect(Missings.replace(main_data2_1min[:Low], 0.0)))
data2_close_1min_i = Float64.(collect(Missings.replace(main_data2_1min[:Close], 0.0)))
# Data 3
data3_date_time_1min_i = main_data3_1min[:Date_Time]
data3_date_1min_i = String.(collect(Missings.replace(main_data3_1min[:Date_index], "")))
data3_time_1min_i = String.(collect(Missings.replace(main_data3_1min[:Time_index], "")))
data3_open_1min_i = Float64.(collect(Missings.replace(main_data3_1min[:Open], 0.0)))
data3_high_1min_i = Float64.(collect(Missings.replace(main_data3_1min[:High], 0.0)))
data3_low_1min_i = Float64.(collect(Missings.replace(main_data3_1min[:Low], 0.0)))
data3_close_1min_i = Float64.(collect(Missings.replace(main_data3_1min[:Close], 0.0)))
# Forward Fill Missing Data 1
# Performs like na.locf
###### Note only forward fill if missing data is on the same day!
data1_date_1min_i
data1_open_1min = zeros(data1_open_1min_i)
data1_high_1min = zeros(data1_high_1min_i)
data1_low_1min = zeros(data1_low_1min_i)
data1_close_1min = zeros(data1_close_1min_i)
i=1
for i in 2:length(data1_close_1min)
if i == 2
# fill index 1
data1_open_1min[i-1] = data1_open_1min_i[i-1]
data1_high_1min[i-1] = data1_high_1min_i[i-1]
data1_low_1min[i-1] = data1_low_1min_i[i-1]
data1_close_1min[i-1] = data1_close_1min_i[i-1]
end
if (data1_close_1min_i[i] == 0.0 && data1_close_1min_i[i-1] != 0.0) & (data1_date_1min_i[i] == data1_date_1min_i[i-1])
data1_open_1min[i] = data1_open_1min_i[i-1]
data1_high_1min[i] = data1_high_1min_i[i-1]
data1_low_1min[i] = data1_low_1min_i[i-1]
data1_close_1min[i] = data1_close_1min_i[i-1]
else
data1_open_1min[i] = data1_open_1min_i[i]
data1_high_1min[i] = data1_high_1min_i[i]
data1_low_1min[i] = data1_low_1min_i[i]
data1_close_1min[i] = data1_close_1min_i[i]
end
if (data1_close_1min_i[i] == 0.0 && data1_close_1min_i[i-1] == 0.0) & (data1_date_1min_i[i] == data1_date_1min_i[i-1])
data1_open_1min[i] = data1_open_1min[i-1]
data1_high_1min[i] = data1_high_1min[i-1]
data1_low_1min[i] = data1_low_1min[i-1]
data1_close_1min[i] = data1_close_1min[i-1]
end
end
# Reverse loop to back fill
# Note julia does not have reverse iteration
data1_open_1min_rev = reverse(data1_open_1min)
data1_high_1min_rev = reverse(data1_high_1min)
data1_low_1min_rev = reverse(data1_low_1min)
data1_close_1min_rev = reverse(data1_close_1min)
# Data 1 reverse
data1_open_1min_rev_out = zeros(data1_open_1min_rev)
data1_high_1min_rev_out = zeros(data1_high_1min_rev)
data1_low_1min_rev_out = zeros(data1_low_1min_rev)
data1_close_1min_rev_out = zeros(data1_close_1min_rev)
i=1
for i in 2:length(data1_close_1min_rev)
if i == 2
# fill index 1
data1_open_1min_rev[i-1] = data1_open_1min_rev[i-1]
data1_high_1min_rev[i-1] = data1_high_1min_rev[i-1]
data1_low_1min_rev[i-1] = data1_low_1min_rev[i-1]
data1_close_1min_rev[i-1] = data1_close_1min_rev[i-1]
end
if (data1_close_1min_rev[i] == 0.0 && data1_close_1min_rev[i-1] != 0.0)
data1_open_1min_rev[i] = data1_open_1min_rev[i-1]
data1_high_1min_rev[i] = data1_high_1min_rev[i-1]
data1_low_1min_rev[i] = data1_low_1min_rev[i-1]
data1_close_1min_rev[i] = data1_close_1min_rev[i-1]
else
data1_open_1min_rev[i] = data1_open_1min_rev[i]
data1_high_1min_rev[i] = data1_high_1min_rev[i]
data1_low_1min_rev[i] = data1_low_1min_rev[i]
data1_close_1min_rev[i] = data1_close_1min_rev[i]
end
if (data1_close_1min_rev[i] == 0.0 && data1_close_1min_rev[i-1] == 0.0)
data1_open_1min_rev[i] = data1_open_1min_rev[i-1]
data1_high_1min_rev[i] = data1_high_1min_rev[i-1]
data1_low_1min_rev[i] = data1_low_1min_rev[i-1]
data1_close_1min_rev[i] = data1_close_1min_rev[i-1]
end
end
# Reverse to normal index position
data1_open_1min = reverse(data1_open_1min_rev)
data1_high_1min = reverse(data1_high_1min_rev)
data1_low_1min = reverse(data1_low_1min_rev)
data1_close_1min = reverse(data1_close_1min_rev)
# Validation
all = hcat(data1_date_1min_i[1:144130],data1_time_1min_i[1:144130],data1_open_1min[1:144130])
all
# Forward Fill Missing Data 2
# Performs like na.locf
###### Note only forward fill if missing data is on the same day!
data2_date_1min_i
data2_open_1min = zeros(data2_open_1min_i)
data2_high_1min = zeros(data2_high_1min_i)
data2_low_1min = zeros(data2_low_1min_i)
data2_close_1min = zeros(data2_close_1min_i)
i=1
for i in 2:length(data2_close_1min)
if i == 2
# fill index 1
data2_open_1min[i-1] = data2_open_1min_i[i-1]
data2_high_1min[i-1] = data2_high_1min_i[i-1]
data2_low_1min[i-1] = data2_low_1min_i[i-1]
data2_close_1min[i-1] = data2_close_1min_i[i-1]
end
if (data2_close_1min_i[i] == 0.0 && data2_close_1min_i[i-1] != 0.0) & (data2_date_1min_i[i] == data2_date_1min_i[i-1])
data2_open_1min[i] = data2_open_1min_i[i-1]
data2_high_1min[i] = data2_high_1min_i[i-1]
data2_low_1min[i] = data2_low_1min_i[i-1]
data2_close_1min[i] = data2_close_1min_i[i-1]
else
data2_open_1min[i] = data2_open_1min_i[i]
data2_high_1min[i] = data2_high_1min_i[i]
data2_low_1min[i] = data2_low_1min_i[i]
data2_close_1min[i] = data2_close_1min_i[i]
end
if (data2_close_1min_i[i] == 0.0 && data2_close_1min_i[i-1] == 0.0) & (data2_date_1min_i[i] == data2_date_1min_i[i-1])
data2_open_1min[i] = data2_open_1min[i-1]
data2_high_1min[i] = data2_high_1min[i-1]
data2_low_1min[i] = data2_low_1min[i-1]
data2_close_1min[i] = data2_close_1min[i-1]
end
end
# Reverse loop to back fill
# Note julia does not have reverse iteration
data2_open_1min_rev = reverse(data2_open_1min)
data2_high_1min_rev = reverse(data2_high_1min)
data2_low_1min_rev = reverse(data2_low_1min)
data2_close_1min_rev = reverse(data2_close_1min)
# Data 1 reverse
data2_open_1min_rev_out = zeros(data2_open_1min_rev)
data2_high_1min_rev_out = zeros(data2_high_1min_rev)
data2_low_1min_rev_out = zeros(data2_low_1min_rev)
data2_close_1min_rev_out = zeros(data2_close_1min_rev)
i=1
for i in 2:length(data2_close_1min_rev)
if i == 2
# fill index 1
data2_open_1min_rev[i-1] = data2_open_1min_rev[i-1]
data2_high_1min_rev[i-1] = data2_high_1min_rev[i-1]
data2_low_1min_rev[i-1] = data2_low_1min_rev[i-1]
data2_close_1min_rev[i-1] = data2_close_1min_rev[i-1]
end
if (data2_close_1min_rev[i] == 0.0 && data2_close_1min_rev[i-1] != 0.0)
data2_open_1min_rev[i] = data2_open_1min_rev[i-1]
data2_high_1min_rev[i] = data2_high_1min_rev[i-1]
data2_low_1min_rev[i] = data2_low_1min_rev[i-1]
data2_close_1min_rev[i] = data2_close_1min_rev[i-1]
else
data2_open_1min_rev[i] = data2_open_1min_rev[i]
data2_high_1min_rev[i] = data2_high_1min_rev[i]
data2_low_1min_rev[i] = data2_low_1min_rev[i]
data2_close_1min_rev[i] = data2_close_1min_rev[i]
end
if (data2_close_1min_rev[i] == 0.0 && data2_close_1min_rev[i-1] == 0.0)
data2_open_1min_rev[i] = data2_open_1min_rev[i-1]
data2_high_1min_rev[i] = data2_high_1min_rev[i-1]
data2_low_1min_rev[i] = data2_low_1min_rev[i-1]
data2_close_1min_rev[i] = data2_close_1min_rev[i-1]
end
end
# Reverse to normal index position
data2_open_1min = reverse(data2_open_1min_rev)
data2_high_1min = reverse(data2_high_1min_rev)
data2_low_1min = reverse(data2_low_1min_rev)
data2_close_1min = reverse(data2_close_1min_rev)
# Forward Fill Missing Data 3
# Performs like na.locf
###### Note only forward fill if missing data is on the same day!
data3_date_1min_i
data3_open_1min = zeros(data3_open_1min_i)
data3_high_1min = zeros(data3_high_1min_i)
data3_low_1min = zeros(data3_low_1min_i)
data3_close_1min = zeros(data3_close_1min_i)
i=1
for i in 2:length(data3_close_1min)
if i == 2
# fill index 1
data3_open_1min[i-1] = data3_open_1min_i[i-1]
data3_high_1min[i-1] = data3_high_1min_i[i-1]
data3_low_1min[i-1] = data3_low_1min_i[i-1]
data3_close_1min[i-1] = data3_close_1min_i[i-1]
end
if (data3_close_1min_i[i] == 0.0 && data3_close_1min_i[i-1] != 0.0) & (data3_date_1min_i[i] == data3_date_1min_i[i-1])
data3_open_1min[i] = data3_open_1min_i[i-1]
data3_high_1min[i] = data3_high_1min_i[i-1]
data3_low_1min[i] = data3_low_1min_i[i-1]
data3_close_1min[i] = data3_close_1min_i[i-1]
else
data3_open_1min[i] = data3_open_1min_i[i]
data3_high_1min[i] = data3_high_1min_i[i]
data3_low_1min[i] = data3_low_1min_i[i]
data3_close_1min[i] = data3_close_1min_i[i]
end
if (data3_close_1min_i[i] == 0.0 && data3_close_1min_i[i-1] == 0.0) & (data3_date_1min_i[i] == data3_date_1min_i[i-1])
data3_open_1min[i] = data3_open_1min[i-1]
data3_high_1min[i] = data3_high_1min[i-1]
data3_low_1min[i] = data3_low_1min[i-1]
data3_close_1min[i] = data3_close_1min[i-1]
end
end
# Reverse loop to back fill
# Note julia does not have reverse iteration
data3_open_1min_rev = reverse(data3_open_1min)
data3_high_1min_rev = reverse(data3_high_1min)
data3_low_1min_rev = reverse(data3_low_1min)
data3_close_1min_rev = reverse(data3_close_1min)
# Data 1 reverse
data3_open_1min_rev_out = zeros(data3_open_1min_rev)
data3_high_1min_rev_out = zeros(data3_high_1min_rev)
data3_low_1min_rev_out = zeros(data3_low_1min_rev)
data3_close_1min_rev_out = zeros(data3_close_1min_rev)
i=1
for i in 2:length(data3_close_1min_rev)
if i == 2
# fill index 1
data3_open_1min_rev[i-1] = data3_open_1min_rev[i-1]
data3_high_1min_rev[i-1] = data3_high_1min_rev[i-1]
data3_low_1min_rev[i-1] = data3_low_1min_rev[i-1]
data3_close_1min_rev[i-1] = data3_close_1min_rev[i-1]
end
if (data3_close_1min_rev[i] == 0.0 && data3_close_1min_rev[i-1] != 0.0)
data3_open_1min_rev[i] = data3_open_1min_rev[i-1]
data3_high_1min_rev[i] = data3_high_1min_rev[i-1]
data3_low_1min_rev[i] = data3_low_1min_rev[i-1]
data3_close_1min_rev[i] = data3_close_1min_rev[i-1]
else
data3_open_1min_rev[i] = data3_open_1min_rev[i]
data3_high_1min_rev[i] = data3_high_1min_rev[i]
data3_low_1min_rev[i] = data3_low_1min_rev[i]
data3_close_1min_rev[i] = data3_close_1min_rev[i]
end
if (data3_close_1min_rev[i] == 0.0 && data3_close_1min_rev[i-1] == 0.0)
data3_open_1min_rev[i] = data3_open_1min_rev[i-1]
data3_high_1min_rev[i] = data3_high_1min_rev[i-1]
data3_low_1min_rev[i] = data3_low_1min_rev[i-1]
data3_close_1min_rev[i] = data3_close_1min_rev[i-1]
end
end
# Reverse to normal index position
data3_open_1min = reverse(data3_open_1min_rev)
data3_high_1min = reverse(data3_high_1min_rev)
data3_low_1min = reverse(data3_low_1min_rev)
data3_close_1min = reverse(data3_close_1min_rev)
# Data 1 find half day holidays
# Note I do not include times when market was closed for
# Find days the market closed early using 30 min data as reference point
hol_ref = CSV.read("C:/Users/Andrew.Bannerman/Desktop/Julia/GE/ES Data/30.min.reg.sess.es.txt", types=[String; String; fill(Float64, 4); Int;Int],header=true)
hol_ref_time = hol_ref[:Time]
hol_ref_date = hol_ref[:Date]
holiday_dates = fill("",length(hol_ref_time))
holiday_times = fill("",length(hol_ref_time))
# Extract dates and last time print
for i = 2:size(holiday_times,1)
if (hol_ref_time[i-1] != "15:00" && hol_ref_time[i] == "09:00")
holiday_times[i-1] = hol_ref_time[i-1]
holiday_dates[i-1] = hol_ref_date[i-1]
end
end
# Late market opens
# http://s3.amazonaws.com/armstrongeconomics-wp/2013/07/NYSE-Closings.pdf
# Only 09/11/2002 Opening delayed until 12:00 noon out of respect for the memorial events commemorating the one-year
# anniversary of the attack on the World Trade Center.
# Other 03/08/1999 was missing 0900 data that TS didnt fill
holiday_dates_late_o = fill("",length(hol_ref_time))
holiday_times_late_o = fill("",length(hol_ref_time))
for i = 2:size(holiday_times,1)
if (hol_ref_time[i-1] == "15:00" && hol_ref_time[i] != "09:00")
holiday_times_late_o[i-1] = hol_ref_time[i]
holiday_dates_late_o[i-1] = hol_ref_date[i]
end
end
# Reduce to remove ""
start_times = holiday_times[holiday_times .!= ""]
holiday_dates = holiday_dates[holiday_dates .!= ""]
end_times = fill("15:00",length(start_times))
# Late opens
dates_late_o = holiday_dates_late_o[holiday_dates_late_o .!= ""]
times_late_o = holiday_times_late_o[holiday_times_late_o .!= ""]
# join date time to one column and concert to datetime format
holiday_date_start_times = String[]
holiday_date_end_times = String[]
temp = String[]
temp1 = String[]
for i in 1:length(holiday_dates)
temp = map(join,zip([holiday_dates[i]],[start_times[i]]), " ")
append!(holiday_date_start_times,temp)
temp1 = map(join,zip([holiday_dates[i]],[end_times[i]]), " ")
append!(holiday_date_end_times,temp1)
end
# Set start and end times as date format
start_date_time = DateTime.(holiday_date_start_times,Dates.DateFormat("mm/dd/yyyy H:M"))
end_date_time = DateTime.(holiday_date_end_times,Dates.DateFormat("mm/dd/yyyy H:M"))
# Late open
# Tag, remember exclude prior to start date, so start is end
#late_open_end_date_time = map(join,zip([dates_late_o[2]],[times_late_o[2]]), " ")
late_open_end_date_time = "09/11/2002 11:00"
late_o_s = "09/11/2002 08:31"
late_open_end_date_time = DateTime(late_open_end_date_time,Dates.DateFormat("mm/dd/yyyy H:M"))
late_open_start_date_time = DateTime(late_o_s,Dates.DateFormat("mm/dd/yyyy H:M"))
# Adjust for holidays
# As indexed from ES daily bars, major closes already included
# This adjusts early closes and late opens
# Data 1
data1_date_1min_i_f = fill("",length(data1_date_1min_i))
data1_time_1min_i_f = fill("",length(data1_time_1min_i))
data1_open_1min_f = zeros(data1_open_1min)
data1_high_1min_f = zeros(data1_high_1min)
data1_low_1min_f = zeros(data1_low_1min)
data1_close_1min_f = zeros(data1_close_1min)
for i in 1:length(data1_close_1min_f)
if (data1_date_time_1min_i[i] <= start_date_time[1]) | (data1_date_time_1min_i[i] > end_date_time[1]) & (data1_date_time_1min_i[i] <= start_date_time[2]) | (data1_date_time_1min_i[i] > end_date_time[2]) &
(data1_date_time_1min_i[i] <= start_date_time[3]) | (data1_date_time_1min_i[i] > end_date_time[3]) & (data1_date_time_1min_i[i] <= start_date_time[4]) | (data1_date_time_1min_i[i] > end_date_time[4]) &
(data1_date_time_1min_i[i] <= start_date_time[5]) | (data1_date_time_1min_i[i] > end_date_time[5]) & (data1_date_time_1min_i[i] <= start_date_time[6]) | (data1_date_time_1min_i[i] > end_date_time[6]) &
(data1_date_time_1min_i[i] <= start_date_time[7]) | (data1_date_time_1min_i[i] > end_date_time[7]) & (data1_date_time_1min_i[i] <= start_date_time[8]) | (data1_date_time_1min_i[i] > end_date_time[8]) &
(data1_date_time_1min_i[i] <= start_date_time[9]) | (data1_date_time_1min_i[i] > end_date_time[9]) & (data1_date_time_1min_i[i] <= start_date_time[10]) | (data1_date_time_1min_i[i] > end_date_time[10]) &
(data1_date_time_1min_i[i] <= start_date_time[11]) | (data1_date_time_1min_i[i] > end_date_time[11]) & (data1_date_time_1min_i[i] <= start_date_time[12]) | (data1_date_time_1min_i[i] > end_date_time[12]) &
(data1_date_time_1min_i[i] <= start_date_time[13]) | (data1_date_time_1min_i[i] > end_date_time[13]) & (data1_date_time_1min_i[i] <= start_date_time[14]) | (data1_date_time_1min_i[i] > end_date_time[14]) &
(data1_date_time_1min_i[i] <= start_date_time[15]) | (data1_date_time_1min_i[i] > end_date_time[15]) & (data1_date_time_1min_i[i] <= start_date_time[16]) | (data1_date_time_1min_i[i] > end_date_time[16]) &
(data1_date_time_1min_i[i] <= start_date_time[17]) | (data1_date_time_1min_i[i] > end_date_time[17]) & (data1_date_time_1min_i[i] <= start_date_time[18]) | (data1_date_time_1min_i[i] > end_date_time[18]) &
(data1_date_time_1min_i[i] <= start_date_time[19]) | (data1_date_time_1min_i[i] > end_date_time[19]) & (data1_date_time_1min_i[i] <= start_date_time[20]) | (data1_date_time_1min_i[i] > end_date_time[20]) &
(data1_date_time_1min_i[i] <= start_date_time[21]) | (data1_date_time_1min_i[i] > end_date_time[21]) & (data1_date_time_1min_i[i] <= start_date_time[22]) | (data1_date_time_1min_i[i] > end_date_time[22]) &
(data1_date_time_1min_i[i] <= start_date_time[23]) | (data1_date_time_1min_i[i] > end_date_time[23]) & (data1_date_time_1min_i[i] <= start_date_time[24]) | (data1_date_time_1min_i[i] > end_date_time[24]) &
(data1_date_time_1min_i[i] <= start_date_time[25]) | (data1_date_time_1min_i[i] > end_date_time[25]) & (data1_date_time_1min_i[i] <= start_date_time[26]) | (data1_date_time_1min_i[i] > end_date_time[26]) &
(data1_date_time_1min_i[i] <= start_date_time[27]) | (data1_date_time_1min_i[i] > end_date_time[27]) & (data1_date_time_1min_i[i] <= start_date_time[28]) | (data1_date_time_1min_i[i] > end_date_time[28]) &
(data1_date_time_1min_i[i] <= start_date_time[29]) | (data1_date_time_1min_i[i] > end_date_time[29]) & (data1_date_time_1min_i[i] <= start_date_time[30]) | (data1_date_time_1min_i[i] > end_date_time[30]) &
(data1_date_time_1min_i[i] <= start_date_time[31]) | (data1_date_time_1min_i[i] > end_date_time[31]) & (data1_date_time_1min_i[i] <= start_date_time[32]) | (data1_date_time_1min_i[i] > end_date_time[32]) &
(data1_date_time_1min_i[i] <= start_date_time[33]) | (data1_date_time_1min_i[i] > end_date_time[33]) & (data1_date_time_1min_i[i] <= start_date_time[34]) | (data1_date_time_1min_i[i] > end_date_time[34]) &
(data1_date_time_1min_i[i] <= start_date_time[35]) | (data1_date_time_1min_i[i] > end_date_time[35]) & (data1_date_time_1min_i[i] <= start_date_time[36]) | (data1_date_time_1min_i[i] > end_date_time[36]) &
(data1_date_time_1min_i[i] <= start_date_time[37]) | (data1_date_time_1min_i[i] > end_date_time[37]) & (data1_date_time_1min_i[i] <= start_date_time[38]) | (data1_date_time_1min_i[i] > end_date_time[38]) &
(data1_date_time_1min_i[i] <= start_date_time[39]) | (data1_date_time_1min_i[i] > end_date_time[39]) & (data1_date_time_1min_i[i] <= start_date_time[40]) | (data1_date_time_1min_i[i] > end_date_time[40]) &
(data1_date_time_1min_i[i] <= start_date_time[41]) | (data1_date_time_1min_i[i] > end_date_time[41]) & (data1_date_time_1min_i[i] <= start_date_time[42]) | (data1_date_time_1min_i[i] > end_date_time[42]) &
(data1_date_time_1min_i[i] <= start_date_time[43]) | (data1_date_time_1min_i[i] > end_date_time[43]) & (data1_date_time_1min_i[i] <= start_date_time[44]) | (data1_date_time_1min_i[i] > end_date_time[44]) &
(data1_date_time_1min_i[i] <= start_date_time[45]) | (data1_date_time_1min_i[i] > end_date_time[45]) & (data1_date_time_1min_i[i] <= start_date_time[46]) | (data1_date_time_1min_i[i] > end_date_time[46]) &
(data1_date_time_1min_i[i] <= start_date_time[47]) | (data1_date_time_1min_i[i] > end_date_time[47]) & (data1_date_time_1min_i[i] <= start_date_time[48]) | (data1_date_time_1min_i[i] > end_date_time[48]) &
(data1_date_time_1min_i[i] <= start_date_time[49]) | (data1_date_time_1min_i[i] > end_date_time[49]) & (data1_date_time_1min_i[i] <= start_date_time[50]) | (data1_date_time_1min_i[i] > end_date_time[50]) &
(data1_date_time_1min_i[i] <= start_date_time[51]) | (data1_date_time_1min_i[i] > end_date_time[51])
# Add all days
data1_date_1min_i_f[i] = data1_date_1min_i[i]
data1_time_1min_i_f[i] = data1_time_1min_i[i]
data1_open_1min_f[i] = data1_open_1min[i]
data1_high_1min_f[i] = data1_high_1min[i]
data1_low_1min_f[i] = data1_low_1min[i]
data1_close_1min_f[i] = data1_close_1min[i]
end
end
# Loop to mark late open with "" 0.0
data1_date_1min_i_filt = fill("",length(data1_date_1min_i))
data1_time_1min_i_filt = fill("",length(data1_time_1min_i))
data1_open_1min_filt = zeros(data1_open_1min)
data1_high_1min_filt = zeros(data1_high_1min)
data1_low_1min_filt = zeros(data1_low_1min)
data1_close_1min_filt = zeros(data1_close_1min)
for i in 1:length(data1_close_1min_filt)
if (data1_date_time_1min_i[i] < late_open_start_date_time) | (data1_date_time_1min_i[i] >= late_open_end_date_time)
data1_date_1min_i_filt[i] = data1_date_1min_i_f[i]
data1_time_1min_i_filt[i] = data1_time_1min_i_f[i]
data1_open_1min_filt[i] = data1_open_1min_f[i]
data1_high_1min_filt[i] = data1_high_1min_f[i]
data1_low_1min_filt[i] = data1_low_1min_f[i]
data1_close_1min_filt[i] = data1_close_1min_f[i]
end
end
# Reduce To Remove all holiday half days ("" or 0.0)
# Data 1
data1_date_1min_i = data1_date_1min_i_filt[data1_date_1min_i_filt .!= ""]
data1_time_1min_i = data1_time_1min_i_filt[data1_time_1min_i_filt .!= ""]
data1_open_1min = data1_open_1min_filt[data1_open_1min_filt .!= 0]
data1_high_1min = data1_high_1min_filt[data1_high_1min_filt .!= 0]
data1_low_1min = data1_low_1min_filt[data1_low_1min_filt .!= 0]
data1_close_1min = data1_close_1min_filt[data1_close_1min_filt .!= 0]
# Validation checks
# Test that the holiday dates have been shortened to half days
#time_out = fill("",length(data1_date_1min_i_filt))
#test_date = data1_date_1min_i_filt[data1_date_1min_i_filt .== "09/11/2002"]
#for i in 1:length(data1_date_1min_i_filt)
#if data1_date_1min_i_filt[i] == "09/11/2002"
# time_out[i] = data1_time_1min_i_filt[i]
#else
# time_out[i] = ""
#end
#end
#time_out = time_out[time_out .!= ""] # reduce to remove 0
# Adjust for holidays
# As indexed from ES daily bars, major closes already included
# This adjusts early closes and late opens
# Data 2
data2_date_1min_i_f = fill("",length(data2_date_1min_i))
data2_time_1min_i_f = fill("",length(data2_time_1min_i))
data2_open_1min_f = zeros(data2_open_1min)
data2_high_1min_f = zeros(data2_high_1min)
data2_low_1min_f = zeros(data2_low_1min)
data2_close_1min_f = zeros(data2_close_1min)
for i in 1:length(data2_close_1min_f)
if (data2_date_time_1min_i[i] <= start_date_time[1]) | (data2_date_time_1min_i[i] > end_date_time[1]) & (data2_date_time_1min_i[i] <= start_date_time[2]) | (data2_date_time_1min_i[i] > end_date_time[2]) &
(data2_date_time_1min_i[i] <= start_date_time[3]) | (data2_date_time_1min_i[i] > end_date_time[3]) & (data2_date_time_1min_i[i] <= start_date_time[4]) | (data2_date_time_1min_i[i] > end_date_time[4]) &
(data2_date_time_1min_i[i] <= start_date_time[5]) | (data2_date_time_1min_i[i] > end_date_time[5]) & (data2_date_time_1min_i[i] <= start_date_time[6]) | (data2_date_time_1min_i[i] > end_date_time[6]) &
(data2_date_time_1min_i[i] <= start_date_time[7]) | (data2_date_time_1min_i[i] > end_date_time[7]) & (data2_date_time_1min_i[i] <= start_date_time[8]) | (data2_date_time_1min_i[i] > end_date_time[8]) &
(data2_date_time_1min_i[i] <= start_date_time[9]) | (data2_date_time_1min_i[i] > end_date_time[9]) & (data2_date_time_1min_i[i] <= start_date_time[10]) | (data2_date_time_1min_i[i] > end_date_time[10]) &
(data2_date_time_1min_i[i] <= start_date_time[11]) | (data2_date_time_1min_i[i] > end_date_time[11]) & (data2_date_time_1min_i[i] <= start_date_time[12]) | (data2_date_time_1min_i[i] > end_date_time[12]) &
(data2_date_time_1min_i[i] <= start_date_time[13]) | (data2_date_time_1min_i[i] > end_date_time[13]) & (data2_date_time_1min_i[i] <= start_date_time[14]) | (data2_date_time_1min_i[i] > end_date_time[14]) &
(data2_date_time_1min_i[i] <= start_date_time[15]) | (data2_date_time_1min_i[i] > end_date_time[15]) & (data2_date_time_1min_i[i] <= start_date_time[16]) | (data2_date_time_1min_i[i] > end_date_time[16]) &
(data2_date_time_1min_i[i] <= start_date_time[17]) | (data2_date_time_1min_i[i] > end_date_time[17]) & (data2_date_time_1min_i[i] <= start_date_time[18]) | (data2_date_time_1min_i[i] > end_date_time[18]) &
(data2_date_time_1min_i[i] <= start_date_time[19]) | (data2_date_time_1min_i[i] > end_date_time[19]) & (data2_date_time_1min_i[i] <= start_date_time[20]) | (data2_date_time_1min_i[i] > end_date_time[20]) &
(data2_date_time_1min_i[i] <= start_date_time[21]) | (data2_date_time_1min_i[i] > end_date_time[21]) & (data2_date_time_1min_i[i] <= start_date_time[22]) | (data2_date_time_1min_i[i] > end_date_time[22]) &
(data2_date_time_1min_i[i] <= start_date_time[23]) | (data2_date_time_1min_i[i] > end_date_time[23]) & (data2_date_time_1min_i[i] <= start_date_time[24]) | (data2_date_time_1min_i[i] > end_date_time[24]) &
(data2_date_time_1min_i[i] <= start_date_time[25]) | (data2_date_time_1min_i[i] > end_date_time[25]) & (data2_date_time_1min_i[i] <= start_date_time[26]) | (data2_date_time_1min_i[i] > end_date_time[26]) &
(data2_date_time_1min_i[i] <= start_date_time[27]) | (data2_date_time_1min_i[i] > end_date_time[27]) & (data2_date_time_1min_i[i] <= start_date_time[28]) | (data2_date_time_1min_i[i] > end_date_time[28]) &
(data2_date_time_1min_i[i] <= start_date_time[29]) | (data2_date_time_1min_i[i] > end_date_time[29]) & (data2_date_time_1min_i[i] <= start_date_time[30]) | (data2_date_time_1min_i[i] > end_date_time[30]) &
(data2_date_time_1min_i[i] <= start_date_time[31]) | (data2_date_time_1min_i[i] > end_date_time[31]) & (data2_date_time_1min_i[i] <= start_date_time[32]) | (data2_date_time_1min_i[i] > end_date_time[32]) &
(data2_date_time_1min_i[i] <= start_date_time[33]) | (data2_date_time_1min_i[i] > end_date_time[33]) & (data2_date_time_1min_i[i] <= start_date_time[34]) | (data2_date_time_1min_i[i] > end_date_time[34]) &
(data2_date_time_1min_i[i] <= start_date_time[35]) | (data2_date_time_1min_i[i] > end_date_time[35]) & (data2_date_time_1min_i[i] <= start_date_time[36]) | (data2_date_time_1min_i[i] > end_date_time[36]) &
(data2_date_time_1min_i[i] <= start_date_time[37]) | (data2_date_time_1min_i[i] > end_date_time[37]) & (data2_date_time_1min_i[i] <= start_date_time[38]) | (data2_date_time_1min_i[i] > end_date_time[38]) &
(data2_date_time_1min_i[i] <= start_date_time[39]) | (data2_date_time_1min_i[i] > end_date_time[39]) & (data2_date_time_1min_i[i] <= start_date_time[40]) | (data2_date_time_1min_i[i] > end_date_time[40]) &
(data2_date_time_1min_i[i] <= start_date_time[41]) | (data2_date_time_1min_i[i] > end_date_time[41]) & (data2_date_time_1min_i[i] <= start_date_time[42]) | (data2_date_time_1min_i[i] > end_date_time[42]) &
(data2_date_time_1min_i[i] <= start_date_time[43]) | (data2_date_time_1min_i[i] > end_date_time[43]) & (data2_date_time_1min_i[i] <= start_date_time[44]) | (data2_date_time_1min_i[i] > end_date_time[44]) &
(data2_date_time_1min_i[i] <= start_date_time[45]) | (data2_date_time_1min_i[i] > end_date_time[45]) & (data2_date_time_1min_i[i] <= start_date_time[46]) | (data2_date_time_1min_i[i] > end_date_time[46]) &
(data2_date_time_1min_i[i] <= start_date_time[47]) | (data2_date_time_1min_i[i] > end_date_time[47]) & (data2_date_time_1min_i[i] <= start_date_time[48]) | (data2_date_time_1min_i[i] > end_date_time[48]) &
(data2_date_time_1min_i[i] <= start_date_time[49]) | (data2_date_time_1min_i[i] > end_date_time[49]) & (data2_date_time_1min_i[i] <= start_date_time[50]) | (data2_date_time_1min_i[i] > end_date_time[50]) &
(data2_date_time_1min_i[i] <= start_date_time[51]) | (data2_date_time_1min_i[i] > end_date_time[51])
# Add all days
data2_date_1min_i_f[i] = data2_date_1min_i[i]
data2_time_1min_i_f[i] = data2_time_1min_i[i]
data2_open_1min_f[i] = data2_open_1min[i]
data2_high_1min_f[i] = data2_high_1min[i]
data2_low_1min_f[i] = data2_low_1min[i]
data2_close_1min_f[i] = data2_close_1min[i]
end
end
# Loop to mark late open with "" 0.0
data2_date_1min_i_filt = fill("",length(data2_date_1min_i))
data2_time_1min_i_filt = fill("",length(data2_time_1min_i))
data2_open_1min_filt = zeros(data2_open_1min)
data2_high_1min_filt = zeros(data2_high_1min)
data2_low_1min_filt = zeros(data2_low_1min)
data2_close_1min_filt = zeros(data2_close_1min)
for i in 1:length(data2_close_1min_filt)
if (data2_date_time_1min_i[i] < late_open_start_date_time) | (data2_date_time_1min_i[i] >= late_open_end_date_time)
data2_date_1min_i_filt[i] = data2_date_1min_i_f[i]
data2_time_1min_i_filt[i] = data2_time_1min_i_f[i]
data2_open_1min_filt[i] = data2_open_1min_f[i]
data2_high_1min_filt[i] = data2_high_1min_f[i]
data2_low_1min_filt[i] = data2_low_1min_f[i]
data2_close_1min_filt[i] = data2_close_1min_f[i]
end
end
# Reduce To Remove all holiday half days ("" or 0.0)
# Data 2
data2_date_1min_i = data2_date_1min_i_filt[data2_date_1min_i_filt .!= ""]
data2_time_1min_i = data2_time_1min_i_filt[data2_time_1min_i_filt .!= ""]
data2_open_1min = data2_open_1min_filt[data2_open_1min_filt .!= 0]
data2_high_1min = data2_high_1min_filt[data2_high_1min_filt .!= 0]
data2_low_1min = data2_low_1min_filt[data2_low_1min_filt .!= 0]
data2_close_1min = data2_close_1min_filt[data2_close_1min_filt .!= 0]
# Validation checks
# Test that the holiday dates have been shortened to half days
#time_out = fill("",length(data2_date_1min_i_filt))
#test_date = data2_date_1min_i_filt[data2_date_1min_i_filt .== "09/11/2002"]
#for i in 1:length(data2_date_1min_i_filt)
#if data2_date_1min_i_filt[i] == "09/11/2002"
# time_out[i] = data2_time_1min_i_filt[i]
#else
# time_out[i] = ""
#end
#end
#time_out = time_out[time_out .!= ""] # reduce to remove 0
# Adjust for holidays
# As indexed from ES daily bars, major closes already included
# This adjusts early closes and late opens
# Data 3
data3_date_1min_i_f = fill("",length(data3_date_1min_i))
data3_time_1min_i_f = fill("",length(data3_time_1min_i))
data3_open_1min_f = zeros(data3_open_1min)
data3_high_1min_f = zeros(data3_high_1min)
data3_low_1min_f = zeros(data3_low_1min)
data3_close_1min_f = zeros(data3_close_1min)
for i in 1:length(data3_close_1min_f)
if (data3_date_time_1min_i[i] <= start_date_time[1]) | (data3_date_time_1min_i[i] > end_date_time[1]) & (data3_date_time_1min_i[i] <= start_date_time[2]) | (data3_date_time_1min_i[i] > end_date_time[2]) &
(data3_date_time_1min_i[i] <= start_date_time[3]) | (data3_date_time_1min_i[i] > end_date_time[3]) & (data3_date_time_1min_i[i] <= start_date_time[4]) | (data3_date_time_1min_i[i] > end_date_time[4]) &
(data3_date_time_1min_i[i] <= start_date_time[5]) | (data3_date_time_1min_i[i] > end_date_time[5]) & (data3_date_time_1min_i[i] <= start_date_time[6]) | (data3_date_time_1min_i[i] > end_date_time[6]) &
(data3_date_time_1min_i[i] <= start_date_time[7]) | (data3_date_time_1min_i[i] > end_date_time[7]) & (data3_date_time_1min_i[i] <= start_date_time[8]) | (data3_date_time_1min_i[i] > end_date_time[8]) &
(data3_date_time_1min_i[i] <= start_date_time[9]) | (data3_date_time_1min_i[i] > end_date_time[9]) & (data3_date_time_1min_i[i] <= start_date_time[10]) | (data3_date_time_1min_i[i] > end_date_time[10]) &
(data3_date_time_1min_i[i] <= start_date_time[11]) | (data3_date_time_1min_i[i] > end_date_time[11]) & (data3_date_time_1min_i[i] <= start_date_time[12]) | (data3_date_time_1min_i[i] > end_date_time[12]) &
(data3_date_time_1min_i[i] <= start_date_time[13]) | (data3_date_time_1min_i[i] > end_date_time[13]) & (data3_date_time_1min_i[i] <= start_date_time[14]) | (data3_date_time_1min_i[i] > end_date_time[14]) &
(data3_date_time_1min_i[i] <= start_date_time[15]) | (data3_date_time_1min_i[i] > end_date_time[15]) & (data3_date_time_1min_i[i] <= start_date_time[16]) | (data3_date_time_1min_i[i] > end_date_time[16]) &
(data3_date_time_1min_i[i] <= start_date_time[17]) | (data3_date_time_1min_i[i] > end_date_time[17]) & (data3_date_time_1min_i[i] <= start_date_time[18]) | (data3_date_time_1min_i[i] > end_date_time[18]) &
(data3_date_time_1min_i[i] <= start_date_time[19]) | (data3_date_time_1min_i[i] > end_date_time[19]) & (data3_date_time_1min_i[i] <= start_date_time[20]) | (data3_date_time_1min_i[i] > end_date_time[20]) &
(data3_date_time_1min_i[i] <= start_date_time[21]) | (data3_date_time_1min_i[i] > end_date_time[21]) & (data3_date_time_1min_i[i] <= start_date_time[22]) | (data3_date_time_1min_i[i] > end_date_time[22]) &
(data3_date_time_1min_i[i] <= start_date_time[23]) | (data3_date_time_1min_i[i] > end_date_time[23]) & (data3_date_time_1min_i[i] <= start_date_time[24]) | (data3_date_time_1min_i[i] > end_date_time[24]) &
(data3_date_time_1min_i[i] <= start_date_time[25]) | (data3_date_time_1min_i[i] > end_date_time[25]) & (data3_date_time_1min_i[i] <= start_date_time[26]) | (data3_date_time_1min_i[i] > end_date_time[26]) &
(data3_date_time_1min_i[i] <= start_date_time[27]) | (data3_date_time_1min_i[i] > end_date_time[27]) & (data3_date_time_1min_i[i] <= start_date_time[28]) | (data3_date_time_1min_i[i] > end_date_time[28]) &
(data3_date_time_1min_i[i] <= start_date_time[29]) | (data3_date_time_1min_i[i] > end_date_time[29]) & (data3_date_time_1min_i[i] <= start_date_time[30]) | (data3_date_time_1min_i[i] > end_date_time[30]) &
(data3_date_time_1min_i[i] <= start_date_time[31]) | (data3_date_time_1min_i[i] > end_date_time[31]) & (data3_date_time_1min_i[i] <= start_date_time[32]) | (data3_date_time_1min_i[i] > end_date_time[32]) &
(data3_date_time_1min_i[i] <= start_date_time[33]) | (data3_date_time_1min_i[i] > end_date_time[33]) & (data3_date_time_1min_i[i] <= start_date_time[34]) | (data3_date_time_1min_i[i] > end_date_time[34]) &
(data3_date_time_1min_i[i] <= start_date_time[35]) | (data3_date_time_1min_i[i] > end_date_time[35]) & (data3_date_time_1min_i[i] <= start_date_time[36]) | (data3_date_time_1min_i[i] > end_date_time[36]) &
(data3_date_time_1min_i[i] <= start_date_time[37]) | (data3_date_time_1min_i[i] > end_date_time[37]) & (data3_date_time_1min_i[i] <= start_date_time[38]) | (data3_date_time_1min_i[i] > end_date_time[38]) &
(data3_date_time_1min_i[i] <= start_date_time[39]) | (data3_date_time_1min_i[i] > end_date_time[39]) & (data3_date_time_1min_i[i] <= start_date_time[40]) | (data3_date_time_1min_i[i] > end_date_time[40]) &
(data3_date_time_1min_i[i] <= start_date_time[41]) | (data3_date_time_1min_i[i] > end_date_time[41]) & (data3_date_time_1min_i[i] <= start_date_time[42]) | (data3_date_time_1min_i[i] > end_date_time[42]) &
(data3_date_time_1min_i[i] <= start_date_time[43]) | (data3_date_time_1min_i[i] > end_date_time[43]) & (data3_date_time_1min_i[i] <= start_date_time[44]) | (data3_date_time_1min_i[i] > end_date_time[44]) &
(data3_date_time_1min_i[i] <= start_date_time[45]) | (data3_date_time_1min_i[i] > end_date_time[45]) & (data3_date_time_1min_i[i] <= start_date_time[46]) | (data3_date_time_1min_i[i] > end_date_time[46]) &
(data3_date_time_1min_i[i] <= start_date_time[47]) | (data3_date_time_1min_i[i] > end_date_time[47]) & (data3_date_time_1min_i[i] <= start_date_time[48]) | (data3_date_time_1min_i[i] > end_date_time[48]) &
(data3_date_time_1min_i[i] <= start_date_time[49]) | (data3_date_time_1min_i[i] > end_date_time[49]) & (data3_date_time_1min_i[i] <= start_date_time[50]) | (data3_date_time_1min_i[i] > end_date_time[50]) &
(data3_date_time_1min_i[i] <= start_date_time[51]) | (data3_date_time_1min_i[i] > end_date_time[51])
# Add all days
data3_date_1min_i_f[i] = data3_date_1min_i[i]
data3_time_1min_i_f[i] = data3_time_1min_i[i]
data3_open_1min_f[i] = data3_open_1min[i]
data3_high_1min_f[i] = data3_high_1min[i]
data3_low_1min_f[i] = data3_low_1min[i]
data3_close_1min_f[i] = data3_close_1min[i]
end
end
# Loop to mark late open with "" 0.0
data3_date_1min_i_filt = fill("",length(data3_date_1min_i))
data3_time_1min_i_filt = fill("",length(data3_time_1min_i))
data3_open_1min_filt = zeros(data3_open_1min)
data3_high_1min_filt = zeros(data3_high_1min)
data3_low_1min_filt = zeros(data3_low_1min)
data3_close_1min_filt = zeros(data3_close_1min)
for i in 1:length(data3_close_1min_filt)
if (data3_date_time_1min_i[i] < late_open_start_date_time) | (data3_date_time_1min_i[i] >= late_open_end_date_time)
data3_date_1min_i_filt[i] = data3_date_1min_i_f[i]
data3_time_1min_i_filt[i] = data3_time_1min_i_f[i]
data3_open_1min_filt[i] = data3_open_1min_f[i]
data3_high_1min_filt[i] = data3_high_1min_f[i]
data3_low_1min_filt[i] = data3_low_1min_f[i]
data3_close_1min_filt[i] = data3_close_1min_f[i]
end
end
# Reduce To Remove all holiday half days ("" or 0.0)
# Data 3
data3_date_1min_i = data3_date_1min_i_filt[data3_date_1min_i_filt .!= ""]
data3_time_1min_i = data3_time_1min_i_filt[data3_time_1min_i_filt .!= ""]
data3_open_1min = data3_open_1min_filt[data3_open_1min_filt .!= 0]
data3_high_1min = data3_high_1min_filt[data3_high_1min_filt .!= 0]
data3_low_1min = data3_low_1min_filt[data3_low_1min_filt .!= 0]
data3_close_1min = data3_close_1min_filt[data3_close_1min_filt .!= 0]
# Validation checks
# Test that the holiday dates have been shortened to half days
#time_out = fill("",length(data3_date_1min_i_filt))
#test_date = data3_date_1min_i_filt[data3_date_1min_i_filt .== "09/11/2002"]
#for i in 1:length(data3_date_1min_i_filt)
#if data3_date_1min_i_filt[i] == "09/11/2002"
# time_out[i] = data3_time_1min_i_filt[i]
#else
# time_out[i] = ""
#end
#end
#time_out = time_out[time_out .!= ""] # reduce to remove 0
# Carry over high, low and open to same index position as the close
# Data 1 index marker
data1_time_index = zeros(size(data1_close_1min,1))
for i = 1:size(data1_close_1min,1)
temp = data1_time_1min_i[i]
if temp[4:5] == "01" || temp[4:5] == "31"
data1_time_index[i] = 1.0
else
data1_time_index[i] = 0.0
end
end
# verify
#all = hcat(data1_time_index[487140:487700],data1_time_1min_i[487140:487700],data1_date_1min_i[487140:487700],data1_open_1min[487140:487700],data1_high_1min[487140:487700],data1_low_1min[487140:487700],data1_close_1min[487140:487700],data1_h_carry[487140:487700],data1_l_carry[487140:487700])
#all = hcat(data1_time_index[1:32],data1_time_1min_i_filt[1:32])
#all = DataFrame(all)
# Data 2 index marker
data2_time_index = zeros(size(data2_close_1min,1))
for i = 1:size(data2_close_1min,1)
temp = data2_time_1min_i[i]
if temp[4:5] == "01" || temp[4:5] == "16" || temp[4:5] == "31" || temp[4:5] == "46"
data2_time_index[i] = 1.0
else
data2_time_index[i] = 0.0
end
end
# Data 3 index marker
data3_time_index = zeros(size(data3_close_1min,1))
for i = 1:size(data3_close_1min,1)
temp = data3_time_1min_i[i]
if temp[4:5] == "01" || temp[4:5] == "11" || temp[4:5] == "21" || temp[4:5] == "31" || temp[4:5] == "41" || temp[4:5] == "51"
data3_time_index[i] = 1.0
else
data3_time_index[i] = 0.0
end
end
# Forward carry high
# Data 1
data1_h_carry = zeros(size(data1_high_1min,1))
temp = data1_high_1min[1] # Set static variable to compare too
tempsave = zeros(data1_high_1min) # Set this to see what is happening with temp
for i = 2:size(data1_high_1min,1) # start on 2nd i as using [i-1]
if data1_high_1min[i] > temp # if current dummy[i] over temp (at start this evaluates to dummy[1]) and temp is again updated later..
data1_h_carry[i] = data1_high_1min[i] # as new high save new high to dummy[i]
temp = data1_h_carry[i] # update temp with new high
tempsave[i] = temp # track temp
else
data1_h_carry[i] = data1_h_carry[i-1] # if not a new high then use previous high found
temp = data1_h_carry[i-1] # update temp with prev. high found
tempsave[i] = temp # track temp
end
if data1_time_index[i] == 1.0 # if time index is 1
temp = data1_high_1min[i] # reset temp with dummy[i]
data1_h_carry[i] = temp # add dummy[i] to running
end
end
# Forward carry low
# Data 1
data1_l_carry = zeros(size(data1_low_1min,1))
temp = data1_low_1min[1] # Set static variable to compare too
tempsave = zeros(data1_low_1min) # Set this to see what is happening with temp
for i = 2:size(data1_low_1min,1) # start on 2nd i as using [i-1]
if data1_low_1min[i] < temp # if current dummy[i] over temp (at start this evaluates to dummy[1]) and temp is again updated later..
data1_l_carry[i] = data1_low_1min[i] # as new high save new high to dummy[i]
temp = data1_l_carry[i] # update temp with new high
tempsave[i] = temp # track temp
else
data1_l_carry[i] = data1_l_carry[i-1] # if not a new high then use previous high found
temp = data1_l_carry[i-1] # update temp with prev. high found
tempsave[i] = temp # track temp
end
if data1_time_index[i] == 1.0 # if time index is 1
temp = data1_low_1min[i] # reset temp with dummy[i]
data1_l_carry[i] = temp # add dummy[i] to running
end
end
# Forward carry high
# Data 2
data2_h_carry = zeros(size(data2_high_1min,1))
temp = data2_high_1min[1] # Set static variable to compare too
tempsave = zeros(data2_high_1min) # Set this to see what is happening with temp
for i = 2:size(data2_high_1min,1) # start on 2nd i as using [i-1]
if data2_high_1min[i] > temp # if current dummy[i] over temp (at start this evaluates to dummy[1]) and temp is again updated later..
data2_h_carry[i] = data2_high_1min[i] # as new high save new high to dummy[i]
temp = data2_h_carry[i] # update temp with new high
tempsave[i] = temp # track temp
else
data2_h_carry[i] = data2_h_carry[i-1] # if not a new high then use previous high found
temp = data2_h_carry[i-1] # update temp with prev. high found
tempsave[i] = temp # track temp
end
if data2_time_index[i] == 1.0 # if time index is 1
temp = data2_high_1min[i] # reset temp with dummy[i]
data2_h_carry[i] = temp # add dummy[i] to running
end
end
# Forward carry low
# Data 2
data2_l_carry = zeros(size(data2_low_1min,1))
temp = data2_low_1min[1] # Set static variable to compare too
tempsave = zeros(data2_low_1min) # Set this to see what is happening with temp
for i = 2:size(data2_low_1min,1) # start on 2nd i as using [i-1]
if data2_low_1min[i] < temp # if current dummy[i] over temp (at start this evaluates to dummy[1]) and temp is again updated later..
data2_l_carry[i] = data2_low_1min[i] # as new high save new high to dummy[i]
temp = data2_l_carry[i] # update temp with new high
tempsave[i] = temp # track temp
else
data2_l_carry[i] = data2_l_carry[i-1] # if not a new high then use previous high found
temp = data2_l_carry[i-1] # update temp with prev. high found
tempsave[i] = temp # track temp
end
if data2_time_index[i] == 1.0 # if time index is 1
temp = data2_low_1min[i] # reset temp with dummy[i]
data2_l_carry[i] = temp # add dummy[i] to running
end
end
# Forward carry high
# Data 3
data3_h_carry = zeros(size(data3_high_1min,1))
temp = data3_high_1min[1] # Set static variable to compare too
tempsave = zeros(data3_high_1min) # Set this to see what is happening with temp
for i = 2:size(data3_high_1min,1) # start on 2nd i as using [i-1]
if data3_high_1min[i] > temp # if current dummy[i] over temp (at start this evaluates to dummy[1]) and temp is again updated later..
data3_h_carry[i] = data3_high_1min[i] # as new high save new high to dummy[i]
temp = data3_h_carry[i] # update temp with new high
tempsave[i] = temp # track temp
else
data3_h_carry[i] = data3_h_carry[i-1] # if not a new high then use previous high found
temp = data3_h_carry[i-1] # update temp with prev. high found
tempsave[i] = temp # track temp
end
if data3_time_index[i] == 1.0 # if time index is 1
temp = data3_high_1min[i] # reset temp with dummy[i]
data3_h_carry[i] = temp # add dummy[i] to running
end
end
# Forward carry low
# Data 3
data3_l_carry = zeros(size(data3_low_1min,1))
temp = data3_low_1min[1] # Set static variable to compare too
tempsave = zeros(data3_low_1min) # Set this to see what is happening with temp
for i = 2:size(data3_low_1min,1) # start on 2nd i as using [i-1]
if data3_low_1min[i] < temp # if current dummy[i] over temp (at start this evaluates to dummy[1]) and temp is again updated later..
data3_l_carry[i] = data3_low_1min[i] # as new high save new high to dummy[i]
temp = data3_l_carry[i] # update temp with new high
tempsave[i] = temp # track temp
else
data3_l_carry[i] = data3_l_carry[i-1] # if not a new high then use previous high found
temp = data3_l_carry[i-1] # update temp with prev. high found
tempsave[i] = temp # track temp
end
if data3_time_index[i] == 1.0 # if time index is 1
temp = data3_low_1min[i] # reset temp with dummy[i]
data3_l_carry[i] = temp # add dummy[i] to running
end
end
# Validate output is as should be
all = hcat(data1_low_1min[1:31],data1_time_index[1:31],data1_l_carry[1:31],tempsave[1:31])
all = DataFrame(all)
print(all)
# Make DateTime
# Thus we can ignore the condition when market opened late 09/11/2002
# Data 1
data1_date_time = String[]
temp = String[]
for i in 1:length(data1_date_1min_i)
temp = map(join,zip([data1_date_1min_i[i]],[data1_time_1min_i[i]]), " ")
append!(data1_date_time,temp)
end
data1_date_time = DateTime.(data1_date_time,Dates.DateFormat("mm/dd/yyyy H:M"))
# Date 2
data2_date_time = String[]
temp = String[]
for i in 1:length(data2_date_1min_i)
temp = map(join,zip([data2_date_1min_i[i]],[data2_time_1min_i[i]]), " ")
append!(data2_date_time,temp)
end
data2_date_time = DateTime.(data2_date_time,Dates.DateFormat("mm/dd/yyyy H:M"))
# Data 3
data3_date_time = String[]
temp = String[]
for i in 1:length(data3_date_1min_i)
temp = map(join,zip([data3_date_1min_i[i]],[data3_time_1min_i[i]]), " ")
append!(data3_date_time,temp)
end
data3_date_time = DateTime.(data3_date_time,Dates.DateFormat("mm/dd/yyyy H:M"))
# Define Data1, Data2, Data3 time samplings
# 30 min, 45min etc..
data1_date = fill("",length(data1_date_1min_i))
data1_time = fill("",length(data1_time_1min_i))
data1_o = zeros(size(data1_close_1min,1))
data1_h = zeros(size(data1_close_1min,1))
data1_l = zeros(size(data1_close_1min,1))
data1_c = zeros(size(data1_close_1min,1))
i=1
for i in 1:length(data1_close_1min)
temp = data1_time_1min_i[i]
if (temp[4:5] == "00" || temp[4:5] == "30") & (data1_date_time[i] != late_open_end_date_time) # set close price at end of 30 minute bar
data1_date[i] = data1_date_1min_i[i] # set date time index to close bar
data1_time[i] = data1_time_1min_i[i] # set date time index to close bar
data1_c[i] = data1_close_1min[i]
data1_h[i] = data1_h_carry[i]
data1_l[i] = data1_l_carry[i]
else
data1_date[i] = ""
data1_time[i] = ""
data1_c[i] = 0.0
end
if temp[4:5] == "01" || temp[4:5] == "31" # Set open price
data1_o[i] = data1_open_1min[i]
else
data1_o[i] = 0.0
end
end
# Data 2
data2_date = fill("",length(data2_date_1min_i))
data2_time = fill("",length(data2_time_1min_i))
data2_o = zeros(size(data2_close_1min,1))
data2_h = zeros(size(data2_close_1min,1))
data2_l = zeros(size(data2_close_1min,1))
data2_c = zeros(size(data2_close_1min,1))
i=1
for i in 1:length(data2_close_1min)
temp = data2_time_1min_i[i]
if (temp[4:5] == "00" || temp[4:5] == "15" || temp[4:5] == "30" || temp[4:5] == "45") & (data2_date_time[i] != late_open_end_date_time) # set close price at end of 30 minute bar
data2_date[i] = data2_date_1min_i[i] # set date time index to close bar
data2_time[i] = data2_time_1min_i[i] # set date time index to close bar
data2_c[i] = data2_close_1min[i]
data2_h[i] = data2_h_carry[i]
data2_l[i] = data2_l_carry[i]
else
data2_date[i] = ""
data2_time[i] = ""
data2_c[i] = 0.0
end
if (temp[4:5] == "01" || temp[4:5] == "16" || temp[4:5] == "31" || temp[4:5] == "46") # Set open price
data2_o[i] = data2_open_1min[i]
else
data2_o[i] = 0.0
end
end
# Data 3
data3_date = fill("",length(data3_date_1min_i))
data3_time = fill("",length(data3_time_1min_i))
data3_o = zeros(size(data3_close_1min,1))
data3_h = zeros(size(data3_close_1min,1))
data3_l = zeros(size(data3_close_1min,1))
data3_c = zeros(size(data3_close_1min,1))
i=1
for i in 1:length(data3_close_1min)
temp = data3_time_1min_i[i]
if (temp[4:5] == "00" || temp[4:5] == "10" || temp[4:5] == "20" || temp[4:5] == "30" || temp[4:5] == "40" || temp[4:5] == "50") & (data3_date_time[i] != late_open_end_date_time)
data3_date[i] = data3_date_1min_i[i] # set date time index to close bar
data3_time[i] = data3_time_1min_i[i] # set date time index to close bar
data3_c[i] = data3_close_1min[i]
data3_h[i] = data3_h_carry[i]
data3_l[i] = data3_l_carry[i]
else
data3_date[i] = ""
data3_time[i] = ""
data3_c[i] = 0.0
end
if (temp[4:5] == "01" || temp[4:5] == "11" || temp[4:5] == "21" || temp[4:5] == "31" || temp[4:5] == "41" || temp[4:5] == "51") # Set open price
data3_o[i] = data3_open_1min[i]
else
data3_o[i] = 0.0
end
end
# Reduce To Remove all 0
# Data 1
data1_date = data1_date[data1_date .!= ""]
data1_time = data1_time[data1_time .!= ""]
data1_o = data1_o[data1_o .!= 0]
data1_h = data1_h[data1_h .!= 0]
data1_l = data1_l[data1_l .!= 0]
data1_c = data1_c[data1_c .!= 0]
# Check open is same length as the HLC
if length(data1_o) != length(data1_c)
fill_no = length(data1_c) - length(data1_o)
data1_o = push!(data1_o,fill_no)
else
data1_o = data1_o
end
# Data 2
data2_date = data2_date[data2_date .!= ""]
data2_time = data2_time[data2_time .!= ""]
data2_o = data2_o[data2_o .!= 0]
data2_h = data2_h[data2_h .!= 0]
data2_l = data2_l[data2_l .!= 0]
data2_c = data2_c[data2_c .!= 0]
# Check open is same length as the HLC
if length(data2_o) != length(data2_c)
fill_no = length(data2_c) - length(data2_o)
data2_o = push!(data2_o,fill_no)
else
data2_o = data2_o
end
# Data 3
data3_date = data3_date[data3_date .!= ""]
data3_time = data3_time[data3_time .!= ""]
data3_o = data3_o[data3_o .!= 0]
data3_h = data3_h[data3_h .!= 0]
data3_l = data3_l[data3_l .!= 0]
data3_c = data3_c[data3_c .!= 0]
# Check open is same length as the HLC
if length(data3_o) != length(data3_c)
fill_no = length(data3_c) - length(data3_o)
data3_o = push!(data3_o,fill_no)
else
data3_o = data3_o
end
# Export as .csv
all = hcat(data3_date,data3_time,data3_o,data3_h,data3_l,data3_c)
all = DataFrame(all)
using DataFrames
CSV.write("C:/Users/Andrew.Bannerman/Desktop/Julia/check_data3_ohlc.csv", all;delim=',')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment