Skip to content

Instantly share code, notes, and snippets.

@flare9x
Last active August 9, 2021 12:23
Show Gist options
  • Save flare9x/a365cce7ab935c5705bad3cbc47edc06 to your computer and use it in GitHub Desktop.
Save flare9x/a365cce7ab935c5705bad3cbc47edc06 to your computer and use it in GitHub Desktop.
NOAA GHCN data is fixed width data - this parses the .dly files and exports as .csv
# Data: https://www.ncei.noaa.gov/access/metadata/landing-page/bin/iso?id=gov.noaa.ncdc%3AC00861
######################################################################################################
# load .dly data file
# load as 1x string per row
# split the string per the readme .txt element / column position (static)
# export as .csv
######################################################################################################
# read all files in directory
all_dly_files = cd(readdir, "C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all/")
all_dly_files_for_export = chop.(all_dly_files, tail = 4)
# initialize output
all_out_tmax = DataFrame(Date = Date[], TMAX = Float64[])
all_out_tmin = DataFrame(Date = Date[], TMIN = Float64[])
all_out_tavg = DataFrame(Date = Date[], TAVG = Float64[])
# start loop
d=1
for d = 1:length(all_dly_files)
print("Starting dly file ", all_dly_files[d]," - Iteration ",d,"\n")
data = CSV.read(join(["C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all/", all_dly_files[d]]), DataFrame, header=false,delim="", ignorerepeated=false)
#data = CSV.read("C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all/ACW00011604.dly", DataFrame, header=false,delim="", ignorerepeated=false)
ID = fill("",size(data,1))
YEAR = fill("",size(data,1))
MONTH = fill("",size(data,1))
ELEMENT = fill("",size(data,1))
VALUE1 = fill(0.0,size(data,1))
MFLAG1 = fill("",size(data,1))
QFLAG1 = fill("",size(data,1))
SFLAG1 = fill("",size(data,1))
VALUE2 = fill(0.0,size(data,1))
MFLAG2 = fill("",size(data,1))
QFLAG2 = fill("",size(data,1))
SFLAG2 = fill("",size(data,1))
VALUE3 = fill(0.0,size(data,1))
MFLAG3 = fill("",size(data,1))
QFLAG3 = fill("",size(data,1))
SFLAG3 = fill("",size(data,1))
VALUE4 = fill(0.0,size(data,1))
MFLAG4 = fill("",size(data,1))
QFLAG4 = fill("",size(data,1))
SFLAG4 = fill("",size(data,1))
VALUE5 = fill(0.0,size(data,1))
MFLAG5 = fill("",size(data,1))
QFLAG5 = fill("",size(data,1))
SFLAG5 = fill("",size(data,1))
VALUE6 = fill(0.0,size(data,1))
MFLAG6 = fill("",size(data,1))
QFLAG6 = fill("",size(data,1))
SFLAG6 = fill("",size(data,1))
VALUE7 = fill(0.0,size(data,1))
MFLAG7 = fill("",size(data,1))
QFLAG7 = fill("",size(data,1))
SFLAG7 = fill("",size(data,1))
VALUE8 = fill(0.0,size(data,1))
MFLAG8 = fill("",size(data,1))
QFLAG8 = fill("",size(data,1))
SFLAG8 = fill("",size(data,1))
VALUE9 = fill(0.0,size(data,1))
MFLAG9 = fill("",size(data,1))
QFLAG9 = fill("",size(data,1))
SFLAG9 = fill("",size(data,1))
VALUE10 = fill(0.0,size(data,1))
MFLAG10 = fill("",size(data,1))
QFLAG10 = fill("",size(data,1))
SFLAG10 = fill("",size(data,1))
VALUE11 = fill(0.0,size(data,1))
MFLAG11 = fill("",size(data,1))
QFLAG11 = fill("",size(data,1))
SFLAG11 = fill("",size(data,1))
VALUE12 = fill(0.0,size(data,1))
MFLAG12 = fill("",size(data,1))
QFLAG12 = fill("",size(data,1))
SFLAG12 = fill("",size(data,1))
VALUE13 = fill(0.0,size(data,1))
MFLAG13 = fill("",size(data,1))
QFLAG13 = fill("",size(data,1))
SFLAG13 = fill("",size(data,1))
VALUE14 = fill(0.0,size(data,1))
MFLAG14 = fill("",size(data,1))
QFLAG14 = fill("",size(data,1))
SFLAG14 = fill("",size(data,1))
VALUE15 = fill(0.0,size(data,1))
MFLAG15 = fill("",size(data,1))
QFLAG15 = fill("",size(data,1))
SFLAG15 = fill("",size(data,1))
VALUE16 = fill(0.0,size(data,1))
MFLAG16 = fill("",size(data,1))
QFLAG16 = fill("",size(data,1))
SFLAG16 = fill("",size(data,1))
VALUE17 = fill(0.0,size(data,1))
MFLAG17 = fill("",size(data,1))
QFLAG17 = fill("",size(data,1))
SFLAG17 = fill("",size(data,1))
VALUE18 = fill(0.0,size(data,1))
MFLAG18 = fill("",size(data,1))
QFLAG18 = fill("",size(data,1))
SFLAG18 = fill("",size(data,1))
VALUE19 = fill(0.0,size(data,1))
MFLAG19 = fill("",size(data,1))
QFLAG19 = fill("",size(data,1))
SFLAG19 = fill("",size(data,1))
VALUE20 = fill(0.0,size(data,1))
MFLAG20 = fill("",size(data,1))
QFLAG20 = fill("",size(data,1))
SFLAG20 = fill("",size(data,1))
VALUE21 = fill(0.0,size(data,1))
MFLAG21 = fill("",size(data,1))
QFLAG21 = fill("",size(data,1))
SFLAG21 = fill("",size(data,1))
VALUE22 = fill(0.0,size(data,1))
MFLAG22 = fill("",size(data,1))
QFLAG22 = fill("",size(data,1))
SFLAG22 = fill("",size(data,1))
VALUE23 = fill(0.0,size(data,1))
MFLAG23 = fill("",size(data,1))
QFLAG23 = fill("",size(data,1))
SFLAG23 = fill("",size(data,1))
VALUE24 = fill(0.0,size(data,1))
MFLAG24 = fill("",size(data,1))
QFLAG24 = fill("",size(data,1))
SFLAG24 = fill("",size(data,1))
VALUE25 = fill(0.0,size(data,1))
MFLAG25 = fill("",size(data,1))
QFLAG25 = fill("",size(data,1))
SFLAG25 = fill("",size(data,1))
VALUE26 = fill(0.0,size(data,1))
MFLAG26 = fill("",size(data,1))
QFLAG26 = fill("",size(data,1))
SFLAG26 = fill("",size(data,1))
VALUE27 = fill(0.0,size(data,1))
MFLAG27 = fill("",size(data,1))
QFLAG27 = fill("",size(data,1))
SFLAG27 = fill("",size(data,1))
VALUE28 = fill(0.0,size(data,1))
MFLAG28 = fill("",size(data,1))
QFLAG28 = fill("",size(data,1))
SFLAG28 = fill("",size(data,1))
VALUE29 = fill(0.0,size(data,1))
MFLAG29 = fill("",size(data,1))
QFLAG29 = fill("",size(data,1))
SFLAG29 = fill("",size(data,1))
VALUE30 = fill(0.0,size(data,1))
MFLAG30 = fill("",size(data,1))
QFLAG30 = fill("",size(data,1))
SFLAG30 = fill("",size(data,1))
VALUE31 = fill(0.0,size(data,1))
MFLAG31 = fill("",size(data,1))
QFLAG31 = fill("",size(data,1))
SFLAG31 = fill("",size(data,1))
i =1
for i = 1:size(data,1)
temp = data[i,1]
ID[i] = temp[1:11]
YEAR[i] = temp[12:15]
MONTH[i] = temp[16:17]
ELEMENT[i] = temp[18:21]
VALUE1[i] = parse.(Float64, temp[22:26]) / 10 # day 1 of the month
MFLAG1[i] = string(temp[27])
QFLAG1[i] = string(temp[28])
SFLAG1[i] = string(temp[29])
VALUE2[i] = parse.(Float64, temp[30:34]) / 10 # day 2 of the month
MFLAG2[i] = string(temp[35])
QFLAG2[i] = string(temp[36])
SFLAG2[i] = string(temp[37])
VALUE3[i] = parse.(Float64, temp[38:42]) / 10 # day 3 of the month
MFLAG3[i] = string(temp[43])
QFLAG3[i] = string(temp[44])
SFLAG3[i] = string(temp[45])
VALUE4[i] = parse.(Float64, temp[46:50]) / 10 # day 4 of the month
MFLAG4[i] = string(temp[51])
QFLAG4[i] = string(temp[52])
SFLAG4[i] = string(temp[53])
VALUE5[i] = parse.(Float64, temp[54:58]) / 10 # day 5 of the month
MFLAG5[i] = string(temp[59])
QFLAG5[i] = string(temp[60])
SFLAG5[i] = string(temp[61])
VALUE6[i] = parse.(Float64, temp[62:66]) / 10 # day 6 of the month
MFLAG6[i] = string(temp[67])
QFLAG6[i] = string(temp[68])
SFLAG6[i] = string(temp[69])
VALUE7[i] = parse.(Float64, temp[70:74]) / 10 # day 7 of the month
MFLAG7[i] = string(temp[75])
QFLAG7[i] = string(temp[76])
SFLAG7[i] = string(temp[77])
VALUE8[i] = parse.(Float64, temp[78:82]) / 10 # day 8 of the month
MFLAG8[i] = string(temp[83])
QFLAG8[i] = string(temp[84])
SFLAG8[i] = string(temp[85])
VALUE9[i] = parse.(Float64, temp[86:90]) / 10 # day 9 of the month
MFLAG9[i] = string(temp[91])
QFLAG9[i] = string(temp[92])
SFLAG9[i] = string(temp[93])
VALUE10[i] = parse.(Float64, temp[94:98]) / 10 # day 10 of the month
MFLAG10[i] = string(temp[99])
QFLAG10[i] = string(temp[100])
SFLAG10[i] = string(temp[101])
VALUE11[i] = parse.(Float64, temp[102:106]) / 10 # day 11 of the month
MFLAG11[i] = string(temp[107])
QFLAG11[i] = string(temp[108])
SFLAG11[i] = string(temp[109])
VALUE12[i] = parse.(Float64, temp[110:114]) / 10 # day 12 of the month
MFLAG12[i] = string(temp[115])
QFLAG12[i] = string(temp[116])
SFLAG12[i] = string(temp[117])
VALUE13[i] = parse.(Float64, temp[118:122]) / 10 # day 13 of the month
MFLAG13[i] = string(temp[123])
QFLAG13[i] = string(temp[124])
SFLAG13[i] = string(temp[125])
VALUE14[i] = parse.(Float64, temp[126:130]) / 10 # day 14 of the month
MFLAG14[i] = string(temp[131])
QFLAG14[i] = string(temp[132])
SFLAG14[i] = string(temp[133])
VALUE15[i] = parse.(Float64, temp[134:138]) / 10 # day 15 of the month
MFLAG15[i] = string(temp[139])
QFLAG15[i] = string(temp[140])
SFLAG15[i] = string(temp[141])
VALUE16[i] = parse.(Float64, temp[142:146]) / 10 # day 16 of the month
MFLAG16[i] = string(temp[147])
QFLAG16[i] = string(temp[148])
SFLAG16[i] = string(temp[149])
VALUE17[i] = parse.(Float64, temp[150:154]) / 10 # day 17 of the month
MFLAG17[i] = string(temp[155])
QFLAG17[i] = string(temp[156])
SFLAG17[i] = string(temp[157])
VALUE18[i] = parse.(Float64, temp[158:162]) / 10 # day 18 of the month
MFLAG18[i] = string(temp[163])
QFLAG18[i] = string(temp[164])
SFLAG18[i] = string(temp[165])
VALUE19[i] = parse.(Float64, temp[166:170]) / 10 # day 19 of the month
MFLAG19[i] = string(temp[171])
QFLAG19[i] = string(temp[172])
SFLAG19[i] = string(temp[173])
VALUE20[i] = parse.(Float64, temp[174:178]) / 10 # day 20 of the month
MFLAG20[i] = string(temp[179])
QFLAG20[i] = string(temp[180])
SFLAG20[i] = string(temp[181])
VALUE21[i] = parse.(Float64, temp[182:186]) / 10 # day 21 of the month
MFLAG21[i] = string(temp[187])
QFLAG21[i] = string(temp[188])
SFLAG21[i] = string(temp[189])
VALUE22[i] = parse.(Float64, temp[190:194]) / 10 # day 22 of the month
MFLAG22[i] = string(temp[195])
QFLAG22[i] = string(temp[196])
SFLAG22[i] = string(temp[197])
VALUE23[i] = parse.(Float64, temp[198:202]) / 10 # day 23 of the month
MFLAG23[i] = string(temp[203])
QFLAG23[i] = string(temp[204])
SFLAG23[i] = string(temp[205])
VALUE24[i] = parse.(Float64, temp[206:210]) / 10 # day 24 of the month
MFLAG24[i] = string(temp[211])
QFLAG24[i] = string(temp[212])
SFLAG24[i] = string(temp[213])
VALUE25[i] = parse.(Float64, temp[214:218]) / 10 / 10 # day 25 of the month
MFLAG25[i] = string(temp[219])
QFLAG25[i] = string(temp[220])
SFLAG25[i] = string(temp[221])
VALUE26[i] = parse.(Float64, temp[222:226]) / 10 / 10 # day 26 of the month
MFLAG26[i] = string(temp[227])
QFLAG26[i] = string(temp[228])
SFLAG26[i] = string(temp[229])
VALUE27[i] = parse.(Float64, temp[230:234]) / 10 # day 27 of the month
MFLAG27[i] = string(temp[235])
QFLAG27[i] = string(temp[236])
SFLAG27[i] = string(temp[237])
VALUE28[i] = parse.(Float64, temp[238:242]) / 10 # day 28 of the month
MFLAG28[i] = string(temp[243])
QFLAG28[i] = string(temp[244])
SFLAG28[i] = string(temp[245])
VALUE29[i] = parse.(Float64, temp[246:250]) / 10 # day 29 of the month
MFLAG29[i] = string(temp[251])
QFLAG29[i] = string(temp[252])
SFLAG29[i] = string(temp[253])
VALUE30[i] = parse.(Float64, temp[254:258]) / 10 # day 30 of the month
MFLAG30[i] = string(temp[259])
QFLAG30[i] = string(temp[260])
SFLAG30[i] = string(temp[261])
VALUE31[i] = parse.(Float64, temp[262:266]) / 10 # day 31 of the month
MFLAG31[i] = string(temp[267])
QFLAG31[i] = string(temp[268])
SFLAG31[i] = string(temp[269])
end
# column names
colnames = ["ID","YEAR","MONTH","ELEMENT","VALUE1","MFLAG1","QFLAG1","SFLAG1","VALUE2","MFLAG2","QFLAG2","SFLAG2","VALUE3","MFLAG3","QFLAG3","SFLAG3","VALUE4","MFLAG4","QFLAG4","SFLAG4","VALUE5",
"MFLAG5","QFLAG5","SFLAG5","VALUE6","MFLAG6","QFLAG6","SFLAG6","VALUE7","MFLAG7","QFLAG7","SFLAG7","VALUE8","MFLAG8","QFLAG8","SFLAG8","VALUE9","MFLAG9","QFLAG9","SFLAG9","VALUE10","MFLAG10",
"QFLAG10","SFLAG10","VALUE11","MFLAG11","QFLAG11","SFLAG11","VALUE12","MFLAG12","QFLAG12","SFLAG12","VALUE13","MFLAG13","QFLAG13","SFLAG13","VALUE14","MFLAG14","QFLAG14","SFLAG14",
"VALUE15","MFLAG15","QFLAG15","SFLAG15","VALUE16","MFLAG16","QFLAG16","SFLAG16","VALUE17","MFLAG17","QFLAG17","SFLAG17","VALUE18","MFLAG18","QFLAG18","SFLAG18","VALUE19","MFLAG19",
"QFLAG19","SFLAG19","VALUE20","MFLAG20","QFLAG20","SFLAG20","VALUE21","MFLAG21","QFLAG21","SFLAG21","VALUE22","MFLAG22","QFLAG22","SFLAG22","VALUE23","MFLAG23","QFLAG23","SFLAG23",
"VALUE24","MFLAG24","QFLAG24","SFLAG24","VALUE25","MFLAG25","QFLAG25","SFLAG25","VALUE26","MFLAG26","QFLAG26","SFLAG26","VALUE27","MFLAG27","QFLAG27","SFLAG27","VALUE28","MFLAG28",
"QFLAG28","SFLAG28","VALUE29","MFLAG29","QFLAG29","SFLAG29","VALUE30","MFLAG30","QFLAG30","SFLAG30","VALUE31","MFLAG31","QFLAG31","SFLAG31"]
# output data frame
out_df = DataFrame(hcat(ID,YEAR,MONTH,ELEMENT,VALUE1,MFLAG1,QFLAG1,SFLAG1,VALUE2,MFLAG2,QFLAG2,SFLAG2,VALUE3,MFLAG3,QFLAG3,SFLAG3,VALUE4,MFLAG4,QFLAG4,SFLAG4,VALUE5,
MFLAG5,QFLAG5,SFLAG5,VALUE6,MFLAG6,QFLAG6,SFLAG6,VALUE7,MFLAG7,QFLAG7,SFLAG7,VALUE8,MFLAG8,QFLAG8,SFLAG8,VALUE9,MFLAG9,QFLAG9,SFLAG9,VALUE10,MFLAG10,
QFLAG10,SFLAG10,VALUE11,MFLAG11,QFLAG11,SFLAG11,VALUE12,MFLAG12,QFLAG12,SFLAG12,VALUE13,MFLAG13,QFLAG13,SFLAG13,VALUE14,MFLAG14,QFLAG14,SFLAG14,
VALUE15,MFLAG15,QFLAG15,SFLAG15,VALUE16,MFLAG16,QFLAG16,SFLAG16,VALUE17,MFLAG17,QFLAG17,SFLAG17,VALUE18,MFLAG18,QFLAG18,SFLAG18,VALUE19,MFLAG19,
QFLAG19,SFLAG19,VALUE20,MFLAG20,QFLAG20,SFLAG20,VALUE21,MFLAG21,QFLAG21,SFLAG21,VALUE22,MFLAG22,QFLAG22,SFLAG22,VALUE23,MFLAG23,QFLAG23,SFLAG23,
VALUE24,MFLAG24,QFLAG24,SFLAG24,VALUE25,MFLAG25,QFLAG25,SFLAG25,VALUE26,MFLAG26,QFLAG26,SFLAG26,VALUE27,MFLAG27,QFLAG27,SFLAG27,VALUE28,MFLAG28,
QFLAG28,SFLAG28,VALUE29,MFLAG29,QFLAG29,SFLAG29,VALUE30,MFLAG30,QFLAG30,SFLAG30,VALUE31,MFLAG31,QFLAG31,SFLAG31))
# rename data frame
rename!(out_df, colnames)
# data statistics
# subset TMAX/TMIN/TAVG Elements
tmax = filter(:ELEMENT => isequal("TMAX"), out_df)
tmin = filter(:ELEMENT => isequal("TMIN"), out_df)
tavg = filter(:ELEMENT => isequal("TAVG"), out_df)
month_index = vcat(1,2,3,4,collect(5:4:128)) # print(vcat(1,2,3,4,collect(5:4:128)))
tmax_mo = tmax[:,[1, 2, 3, 4, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125]]
tmin_mo = tmin[:,[1, 2, 3, 4, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125]]
tavg_mo = tavg[:,[1, 2, 3, 4, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125]]
# build date index for the set
days_total = 31
#day_index = repeat(vcat(["01","02","03","04","05","06","07","08","09"],string.(collect(10:1:31))),inner = size(tmax,1))
day_index = vcat(["01","02","03","04","05","06","07","08","09"],string.(collect(10:1:days_total)))
# Loop through creating date index for the data and pull the TMAX temperature data
dims_tmax = size(tmax,1) * days_total
out_tmax = fill("", dims_tmax) # push to
data_out_tmax = zeros(dims_tmax) # push to
data_grab = [] # temp for loop
temp = [] # temp for loop
count = 0
count1 = 0
for j = 5:size(tmax_mo,2) # loop along columns
#print("this is fucking JJJ",j,"yee\n")
count = count +1
for i = 1 :size(tmax_mo,1) # loop down the rows
count1 = count1+1
if tmax_mo[i,3] == "02" && (day_index[count] == "29" || day_index[count] == "30" || day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmax_mo[i,3] == "04" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmax_mo[i,3] == "06" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmax_mo[i,3] == "09" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmax_mo[i,3] == "11" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmax_mo[i,3] != "01" || tmax_mo[i,3] != "04" || tmax_mo[i,3] != "06" || tmax_mo[i,3] != "09" || tmax_mo[i,3] != "11"
temp = join([tmax_mo[i,2],tmax_mo[i,3],day_index[count]])
data_grab = tmax_mo[i,j]
end
#push!(out_tmax,temp)
#push!(data_out_tmax,data_grab)
out_tmax[count1] = temp
data_out_tmax[count1] = data_grab
end
end
dims_tmin = size(tmin,1) * days_total
out_tmin = fill("", dims_tmin) # push to
data_out_tmin = zeros(dims_tmin) # push to
data_grab = [] # temp for loop
temp = [] # temp for loop
count = 0
count1 = 0
for j = 5:size(tmin_mo,2) # loop along columns
#print("this is fucking JJJ",j,"yee\n")
count = count +1
for i = 1 :size(tmin_mo,1) # loop down the rows
count1 = count1+1
if tmin_mo[i,3] == "02" && (day_index[count] == "29" || day_index[count] == "30" || day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmin_mo[i,3] == "04" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmin_mo[i,3] == "06" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmin_mo[i,3] == "09" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmin_mo[i,3] == "11" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tmin_mo[i,3] != "01" || tmin_mo[i,3] != "04" || tmin_mo[i,3] != "06" || tmin_mo[i,3] != "09" || tmin_mo[i,3] != "11"
temp = join([tmin_mo[i,2],tmin_mo[i,3],day_index[count]])
data_grab = tmin_mo[i,j]
end
out_tmin[count1] = temp
data_out_tmin[count1] = data_grab
end
end
dims_avg = size(tavg,1) * days_total
if dims_avg != 0
out_tavg = fill("", dims_avg) # push to
data_out_tavg = zeros(dims_avg) # push to
data_grab = [] # temp for loop
temp = [] # temp for loop
count = 0
count1 = 0
for j = 5:size(tavg_mo,2) # loop along columns
#print("this is fucking JJJ",j,"yee\n")
count = count +1
for i = 1 :size(tavg_mo,1) # loop down the rows
count1 = count1+1
if tavg_mo[i,3] == "02" && (day_index[count] == "29" || day_index[count] == "30" || day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tavg_mo[i,3] == "04" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tavg_mo[i,3] == "06" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tavg_mo[i,3] == "09" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tavg_mo[i,3] == "11" && (day_index[count] == "31")
temp = ""
data_grab = 99999.999
elseif tavg_mo[i,3] != "01" || tavg_mo[i,3] != "04" || tavg_mo[i,3] != "06" || tavg_mo[i,3] != "09" || tavg_mo[i,3] != "11"
temp = join([tavg_mo[i,2],tavg_mo[i,3],day_index[count]])
data_grab = tavg_mo[i,j]
end
out_tavg[count1] = temp
data_out_tavg[count1] = data_grab
end
end
end
# subset to remove missing data for the shorter months
out_tmax = out_tmax[out_tmax .!= ""] #
data_out_tmax = data_out_tmax[data_out_tmax .!= 99999.999]
out_tmin = out_tmin[out_tmin .!= ""] #
data_out_tmin = Float64.(data_out_tmin[data_out_tmin .!= 99999.999])
if dims_avg != 0
out_tavg = out_tavg[out_tavg .!= ""] #
data_out_tavg = Float64.(data_out_tavg[data_out_tavg .!= 99999.999])
end
# convert date to date format
out_tmax = Date.(out_tmax, "yyyymmdd")
out_tmin = Date.(out_tmin, "yyyymmdd")
if dims_avg != 0
out_tavg = Date.(out_tavg, "yyyymmdd")
end
# create final ordered by date dataframe
out_tmax = DataFrame(Date = out_tmax, TMAX = data_out_tmax)
final_tmax = sort(out_tmax,[:Date])
out_tmin = DataFrame(Date = out_tmin, TMIN = data_out_tmin)
final_tmin = sort(out_tmin,[:Date])
if dims_avg != 0
out_tavg = DataFrame(Date = out_tavg, TAVG = data_out_tavg)
final_tavg = sort(out_tavg,[:Date])
end
#######################################################
# left join to date index
#######################################################
#=test_date = final_tmax[1:5,1]
dummy_temp = [1.0,1.0,1.0,1.0,1.0]
df1 = DataFrame(Date = test_date, TMAX = dummy_temp)
global date_index_tmax = leftjoin(date_index_tmax, final_tmax, on = :Date, makeunique=true)
global date_index_tmin = leftjoin(date_index_tmin, final_tmin, on = :Date, makeunique=true)
=#
# Build output data frames (all parsed from fixed width .dly file)
#global all_out_tmax = vcat(all_out_tmax,final_tmax)
#global all_out_tmin = vcat(all_out_tmin,final_tmin)
#global all_out_tavg = vcat(all_out_tavg,final_tavg)
# write parsed .csv files to file
CSV.write(join(["C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all_csv/tmax/",all_dly_files_for_export[d],".csv"]),final_tmax)
CSV.write(join(["C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all_csv/tmin/",all_dly_files_for_export[d],".csv"]),final_tmin)
if dims_avg != 0
CSV.write(join(["C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all_csv/tavg/",all_dly_files_for_export[d],".csv"]),final_tavg)
end
end # loop
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment