Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
SAPFLUXNET Data bugs fixing for 0.1.3 version
######################################################################
# CODE TO FIX KNOWN BUGS IN THE 0.1.3 VERSION OF THE SAPFLUXNET
# DATABASE
#
# INSTRUCTIONS:
# 1. COPY THIS FILE TO AN EMPTY R SCRIPT IN YOUR LOCAL MACHINE
# 2. INSTALL tidyverse AND sapfluxnetr IF NEEDED
# 3. CHANGE LAST LINE AS DESIRED
# 4. EXECUTE ALL THE SCRIPT
#######################################################################
# install.packages(tidyverse)
# install.packages(sapfluxnetr)
library(tidyverse)
library(sapfluxnetr)
## Site functions ####
ESP_YUN_T3_THI_fixes <- function(folder, csv = FALSE) {
if (!file.exists(file.path(folder, 'ESP_YUN_T3_THI.RData'))) {
return()
}
bad_site <- read_sfn_data('ESP_YUN_T3_THI', folder)
# check sapflow
if ('ESP_YUN_T3_THI_Api_Js_1' %in% names(get_sapf_data(bad_site))) {
good_sapf_data <- bad_site %>%
get_sapf_data()
} else {
good_sapf_data <- bad_site %>%
get_sapf_data() %>%
dplyr::select(
ESP_YUN_T3_THI_Api_Js_1 = ESP_YUN_T3_Api_Js_1,
ESP_YUN_T3_THI_Api_Js_2 = ESP_YUN_T3_Api_Js_2,
ESP_YUN_T3_THI_Api_Js_3 = ESP_YUN_T3_Api_Js_3,
ESP_YUN_T3_THI_Api_Js_4 = ESP_YUN_T3_Api_Js_4,
ESP_YUN_T3_THI_Api_Js_5 = ESP_YUN_T3_Api_Js_5,
ESP_YUN_T3_THI_Api_Js_6 = ESP_YUN_T3_Api_Js_6
)
}
# check flags
if ('ESP_YUN_T3_THI_Api_Js_1' %in% names(get_sapf_flags(bad_site))) {
good_sapf_flags <- bad_site %>%
get_sapf_flags()
} else {
good_sapf_flags <- bad_site %>%
get_sapf_flags() %>%
dplyr::select(
ESP_YUN_T3_THI_Api_Js_1 = ESP_YUN_T3_Api_Js_1,
ESP_YUN_T3_THI_Api_Js_2 = ESP_YUN_T3_Api_Js_2,
ESP_YUN_T3_THI_Api_Js_3 = ESP_YUN_T3_Api_Js_3,
ESP_YUN_T3_THI_Api_Js_4 = ESP_YUN_T3_Api_Js_4,
ESP_YUN_T3_THI_Api_Js_5 = ESP_YUN_T3_Api_Js_5,
ESP_YUN_T3_THI_Api_Js_6 = ESP_YUN_T3_Api_Js_6
)
}
# check plant names in plant metadata
is_plant_md_ok <- bad_site %>%
get_plant_md() %>%
pull(pl_code) %>%
str_detect(pattern = '_THI_') %>%
all()
if (is_plant_md_ok) {
good_plant_md <- bad_site %>%
get_plant_md()
} else {
good_plant_md <- bad_site %>%
get_plant_md() %>%
dplyr::mutate(
pl_code = stringr::str_replace(pl_code, 'ESP_YUN_T3_', 'ESP_YUN_T3_THI_')
)
}
# finaly build the fixed site
ESP_YUN_T3_THI <- bad_site
get_sapf_data(ESP_YUN_T3_THI) <- good_sapf_data
get_sapf_flags(ESP_YUN_T3_THI) <- good_sapf_flags
get_plant_md(ESP_YUN_T3_THI) <- good_plant_md
# and save it
save(ESP_YUN_T3_THI, file = file.path(folder, 'ESP_YUN_T3_THI.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(ESP_YUN_T3_THI) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_YUN_T3_THI)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_YUN_T3_THI_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(ESP_YUN_T3_THI) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_YUN_T3_THI)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_YUN_T3_THI_sapf_flags.csv'
))
# plant_md
get_plant_md(ESP_YUN_T3_THI) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_YUN_T3_THI_plant_md.csv'
))
}
}
COL_MAC_SAF_RAD_fixes <- function(folder, csv = FALSE) {
if (!file.exists(file.path(folder, 'COL_MAC_SAF_RAD.RData'))) {
return()
}
bad_site <- read_sfn_data('COL_MAC_SAF_RAD', folder)
# check sapflow
if ('COL_MAC_SAF_RAD_Tca_Js_1' %in% names(get_sapf_data(bad_site))) {
good_sapf_data <- bad_site %>%
get_sapf_data()
} else {
good_sapf_data <- bad_site %>%
get_sapf_data() %>%
dplyr::select(
COL_MAC_SAF_RAD_Tca_Js_1 = COL_MAC_SAF_Tca_Js_1,
COL_MAC_SAF_RAD_Tca_Js_2 = COL_MAC_SAF_Tca_Js_2,
COL_MAC_SAF_RAD_Tca_Js_3 = COL_MAC_SAF_Tca_Js_3
)
}
# check flags
if ('COL_MAC_SAF_RAD_Tca_Js_1' %in% names(get_sapf_flags(bad_site))) {
good_sapf_flags <- bad_site %>%
get_sapf_flags()
} else {
good_sapf_flags <- bad_site %>%
get_sapf_flags() %>%
dplyr::select(
COL_MAC_SAF_RAD_Tca_Js_1 = COL_MAC_SAF_Tca_Js_1,
COL_MAC_SAF_RAD_Tca_Js_2 = COL_MAC_SAF_Tca_Js_2,
COL_MAC_SAF_RAD_Tca_Js_3 = COL_MAC_SAF_Tca_Js_3
)
}
# check plant names in plant metadata
is_plant_md_ok <- bad_site %>%
get_plant_md() %>%
pull(pl_code) %>%
str_detect(pattern = '_RAD_') %>%
all()
if (is_plant_md_ok) {
good_plant_md <- bad_site %>%
get_plant_md()
} else {
good_plant_md <- bad_site %>%
get_plant_md() %>%
dplyr::mutate(
pl_code = stringr::str_replace(pl_code, 'COL_MAC_SAF_', 'COL_MAC_SAF_RAD_')
)
}
# finaly build the fixed site
COL_MAC_SAF_RAD <- bad_site
get_sapf_data(COL_MAC_SAF_RAD) <- good_sapf_data
get_sapf_flags(COL_MAC_SAF_RAD) <- good_sapf_flags
get_plant_md(COL_MAC_SAF_RAD) <- good_plant_md
# and save it
save(COL_MAC_SAF_RAD, file = file.path(folder, 'COL_MAC_SAF_RAD.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(COL_MAC_SAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(COL_MAC_SAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'COL_MAC_SAF_RAD_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(COL_MAC_SAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(COL_MAC_SAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'COL_MAC_SAF_RAD_sapf_flags.csv'
))
# plant_md
get_plant_md(COL_MAC_SAF_RAD) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'COL_MAC_SAF_RAD_plant_md.csv'
))
}
}
AUS_MAR_HSW_HIG_fixes <- function(folder, csv = FALSE) {
if (!file.exists(file.path(folder, 'AUS_MAR_HSW_HIG.RData'))) {
return()
}
bad_site <- read_sfn_data('AUS_MAR_HSW_HIG', folder)
# check sapflow
if ('AUS_MAR_HSW_HIG_Eob_Js_1' %in% names(get_sapf_data(bad_site))) {
good_sapf_data <- bad_site %>%
get_sapf_data()
} else {
good_sapf_data <- bad_site %>%
get_sapf_data() %>%
dplyr::select(
AUS_MAR_HSW_HIG_Eob_Js_1 = AUS_MAR_HSW_Eob_Js_1,
AUS_MAR_HSW_HIG_Eob_Js_2 = AUS_MAR_HSW_Eob_Js_2,
AUS_MAR_HSW_HIG_Eob_Js_3 = AUS_MAR_HSW_Eob_Js_3,
AUS_MAR_HSW_HIG_Ecy_Js_4 = AUS_MAR_HSW_Ecy_Js_4
)
}
# check flags
if ('AUS_MAR_HSW_HIG_Eob_Js_1' %in% names(get_sapf_flags(bad_site))) {
good_sapf_flags <- bad_site %>%
get_sapf_flags()
} else {
good_sapf_flags <- bad_site %>%
get_sapf_flags() %>%
dplyr::select(
AUS_MAR_HSW_HIG_Eob_Js_1 = AUS_MAR_HSW_Eob_Js_1,
AUS_MAR_HSW_HIG_Eob_Js_2 = AUS_MAR_HSW_Eob_Js_2,
AUS_MAR_HSW_HIG_Eob_Js_3 = AUS_MAR_HSW_Eob_Js_3,
AUS_MAR_HSW_HIG_Ecy_Js_4 = AUS_MAR_HSW_Ecy_Js_4
)
}
# check plant names in plant metadata
is_plant_md_ok <- bad_site %>%
get_plant_md() %>%
pull(pl_code) %>%
str_detect(pattern = '_HIG_') %>%
all()
if (is_plant_md_ok) {
good_plant_md <- bad_site %>%
get_plant_md()
} else {
good_plant_md <- bad_site %>%
get_plant_md() %>%
dplyr::mutate(
pl_code = stringr::str_replace(pl_code, 'AUS_MAR_HSW_', 'AUS_MAR_HSW_HIG_')
)
}
# finaly build the fixed site
AUS_MAR_HSW_HIG <- bad_site
get_sapf_data(AUS_MAR_HSW_HIG) <- good_sapf_data
get_sapf_flags(AUS_MAR_HSW_HIG) <- good_sapf_flags
get_plant_md(AUS_MAR_HSW_HIG) <- good_plant_md
# and save it
save(AUS_MAR_HSW_HIG, file = file.path(folder, 'AUS_MAR_HSW_HIG.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(AUS_MAR_HSW_HIG) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(AUS_MAR_HSW_HIG)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'AUS_MAR_HSW_HIG_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(AUS_MAR_HSW_HIG) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(AUS_MAR_HSW_HIG)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'AUS_MAR_HSW_HIG_sapf_flags.csv'
))
# plant_md
get_plant_md(AUS_MAR_HSW_HIG) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'AUS_MAR_HSW_HIG_plant_md.csv'
))
}
}
USA_SMI_SER_fixes <- function(folder, csv = FALSE) {
if (!file.exists(file.path(folder, 'USA_SMI_SER.RData'))) {
return()
}
bad_site <- read_sfn_data('USA_SMI_SER', folder)
# change the units in sapf data only if we are in plant level, if not, we only need
# to change the metadata, as the sapwood values were the original ones.
if (stringr::str_detect(folder, 'plant')) {
USA_SMI_SER <- bad_site %>%
sfn_mutate_at(
.vars = vars(one_of(names(get_sapf_data(bad_site)[,-1]))),
.funs = list(~ .x * 1e4)
)
} else {
USA_SMI_SER <- bad_site
}
# check plant md
good_plant_md <- USA_SMI_SER %>%
get_plant_md() %>%
mutate(pl_sapw_area = pl_sapw_area * 1e4)
# finaly build the fixed site
get_plant_md(USA_SMI_SER) <- good_plant_md
# and save it
save(USA_SMI_SER, file = file.path(folder, 'USA_SMI_SER.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(USA_SMI_SER) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_SMI_SER)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_SMI_SER_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(USA_SMI_SER) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_SMI_SER)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_SMI_SER_sapf_flags.csv'
))
# plant_md
get_plant_md(USA_SMI_SER) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_SMI_SER_plant_md.csv'
))
}
}
USA_SMI_SCB_fixes <- function(folder, csv = FALSE) {
if (!file.exists(file.path(folder, 'USA_SMI_SCB.RData'))) {
return()
}
bad_site <- read_sfn_data('USA_SMI_SCB', folder)
# change the units in sapf data only if we are in plant level, if not, we only need
# to change the metadata, as the sapwood values were the original ones.
if (stringr::str_detect(folder, 'plant')) {
USA_SMI_SCB <- bad_site %>%
sfn_mutate_at(
.vars = vars(one_of(names(get_sapf_data(bad_site)[,-1]))),
.funs = list(~ .x * 1e4)
)
} else {
USA_SMI_SCB <- bad_site
}
# check plant md
good_plant_md <- USA_SMI_SCB %>%
get_plant_md() %>%
mutate(pl_sapw_area = pl_sapw_area * 1e4)
# finaly build the fixed site
get_plant_md(USA_SMI_SCB) <- good_plant_md
# and save it
save(USA_SMI_SCB, file = file.path(folder, 'USA_SMI_SCB.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(USA_SMI_SCB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_SMI_SCB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_SMI_SCB_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(USA_SMI_SCB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_SMI_SCB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_SMI_SCB_sapf_flags.csv'
))
# plant_md
get_plant_md(USA_SMI_SCB) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_SMI_SCB_plant_md.csv'
))
}
}
USA_CHE_ASP_fixes <- function(folder, csv = FALSE) {
if (!file.exists(file.path(folder, 'USA_CHE_ASP.RData'))) {
return()
}
USA_CHE_ASP <- read_sfn_data('USA_CHE_ASP', folder)
# In this case, there is only to fix the metadata. Sapwood depth is not used to
# calculate sapwood area, as this value is already provided.
good_plant_md <- USA_CHE_ASP %>%
get_plant_md() %>%
mutate(pl_sapw_depth = pl_sapw_depth / 10)
# finaly build the fixed site
get_plant_md(USA_CHE_ASP) <- good_plant_md
# and save it
save(USA_CHE_ASP, file = file.path(folder, 'USA_CHE_ASP.RData'))
# csv files
if (isTRUE(csv)) {
# plant_md
get_plant_md(USA_CHE_ASP) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_CHE_ASP_plant_md.csv'
))
}
}
RUS_POG_VAR_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, 'RUS_POG_VAR.RData'))) {
return()
}
# Here we need to set stand basal area percentage to NA
RUS_POG_VAR <- read_sfn_data('RUS_POG_VAR', folder)
good_species_md <- RUS_POG_VAR %>%
get_species_md() %>%
mutate(sp_basal_area_perc = NA_integer_)
# finaly build the fixed site
get_species_md(RUS_POG_VAR) <- good_species_md
# and save it
save(RUS_POG_VAR, file = file.path(folder, 'RUS_POG_VAR.RData'))
# csv files
if (isTRUE(csv)) {
# species_md
get_species_md(RUS_POG_VAR) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'RUS_POG_VAR_species_md.csv'
))
}
}
## TZ issues fixes ####
# sites <- c(
# 'ZAF_SOU_SOU', 'ZAF_RAD', 'ZAF_FRA_FRA', 'RUS_FYO', 'IDN_JAM_RUB',
# "IDN_JAM_OIL", 'ESP_PRA', "ESP_LAS", 'USA_WVF', 'USA_TNB',
# 'USA_TNO', 'USA_TNP', 'USA_TNY', 'USA_PAR_FER', 'USA_INM',
# 'COL_MAC_SAF_RAD'
# )
#
# good_tz <- c(
# "Etc/GMT-2", "Etc/GMT-2", "Etc/GMT-2", "Etc/GMT-3", "Etc/GMT-7",
# "Etc/GMT-7", "Etc/GMT-1", "Etc/GMT+0", "Etc/GMT+5", "Etc/GMT+5",
# "Etc/GMT+5", "Etc/GMT+5", "Etc/GMT+5", "Etc/GMT+5", "Etc/GMT+5",
# "Etc/GMT+5"
# )
solar_timestamp_creator <- function(timestamp, lat, long) {
mst <- solaR::local2Solar(timestamp, long)
# 2.2.1 warning if solartimestamp has repeated values (due to rounding)
if (length(mst) != length(unique(mst))) {
warning('solar mean time generates repeated timestamps. ',
'Please revise the original timestamp for repeated values.')
}
# STEP 3
# Calculating Apparent Solar Time (Mean Solar Time + Equation of Time)
# 2.1 Equation of time
solD <- solaR::fSolD(lat, mst)
EoT <- solaR::r2sec(solD$EoT)
ast <- lapply(as.Date(strptime(zoo::index(EoT), format = '%Y-%m-%d')),
function(id, vect)
(vect[as.Date(vect) == id] +
zoo::coredata(EoT)[which(as.Date(strptime(zoo::index(EoT),
format = '%Y-%m-%d')) == id)]),
vect = mst)
ast <- do.call("c", ast)
return(ast)
}
ZAF_SOU_SOU_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('ZAF_SOU_SOU','.RData')))) {
return()
}
ZAF_SOU_SOU <- read_sfn_data('ZAF_SOU_SOU', folder)
site_metadata <- get_site_md(ZAF_SOU_SOU)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- ZAF_SOU_SOU %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT-2")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- ZAF_SOU_SOU %>%
get_env_md() %>%
mutate(env_time_zone = "18UTC+02:00, B")
get_timestamp(ZAF_SOU_SOU) <- good_timestamp
get_solar_timestamp(ZAF_SOU_SOU) <- good_solar_timestamp
get_env_md(ZAF_SOU_SOU) <- good_env_md
save(ZAF_SOU_SOU, file = file.path(folder, 'ZAF_SOU_SOU.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(ZAF_SOU_SOU) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_SOU_SOU)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_SOU_SOU_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(ZAF_SOU_SOU) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_SOU_SOU)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_SOU_SOU_sapf_flags.csv'
))
# env_data
get_env_data(ZAF_SOU_SOU) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_SOU_SOU)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_SOU_SOU_env_data.csv'
))
# env_flags
get_env_flags(ZAF_SOU_SOU) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_SOU_SOU)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_SOU_SOU_env_flags.csv'
))
# env_md
get_env_md(ZAF_SOU_SOU) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_SOU_SOU_env_md.csv'
))
}
}
ZAF_RAD_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('ZAF_RAD','.RData')))) {
return()
}
ZAF_RAD <- read_sfn_data('ZAF_RAD', folder)
site_metadata <- get_site_md(ZAF_RAD)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- ZAF_RAD %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT-2")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- ZAF_RAD %>%
get_env_md() %>%
mutate(env_time_zone = "18UTC+02:00, B")
get_timestamp(ZAF_RAD) <- good_timestamp
get_solar_timestamp(ZAF_RAD) <- good_solar_timestamp
get_env_md(ZAF_RAD) <- good_env_md
save(ZAF_RAD, file = file.path(folder, 'ZAF_RAD.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(ZAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_RAD_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(ZAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_RAD_sapf_flags.csv'
))
# env_data
get_env_data(ZAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_RAD_env_data.csv'
))
# env_flags
get_env_flags(ZAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_RAD_env_flags.csv'
))
# env_md
get_env_md(ZAF_RAD) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_RAD_env_md.csv'
))
}
}
ZAF_FRA_FRA_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('ZAF_FRA_FRA','.RData')))) {
return()
}
ZAF_FRA_FRA <- read_sfn_data('ZAF_FRA_FRA', folder)
site_metadata <- get_site_md(ZAF_FRA_FRA)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- ZAF_FRA_FRA %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT-2")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- ZAF_FRA_FRA %>%
get_env_md() %>%
mutate(env_time_zone = "18UTC+02:00, B")
get_timestamp(ZAF_FRA_FRA) <- good_timestamp
get_solar_timestamp(ZAF_FRA_FRA) <- good_solar_timestamp
get_env_md(ZAF_FRA_FRA) <- good_env_md
save(ZAF_FRA_FRA, file = file.path(folder, 'ZAF_FRA_FRA.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(ZAF_FRA_FRA) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_FRA_FRA)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_FRA_FRA_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(ZAF_FRA_FRA) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_FRA_FRA)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_FRA_FRA_sapf_flags.csv'
))
# env_data
get_env_data(ZAF_FRA_FRA) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_FRA_FRA)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_FRA_FRA_env_data.csv'
))
# env_flags
get_env_flags(ZAF_FRA_FRA) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ZAF_FRA_FRA)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_FRA_FRA_env_flags.csv'
))
# env_md
get_env_md(ZAF_FRA_FRA) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ZAF_FRA_FRA_env_md.csv'
))
}
}
RUS_FYO_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('RUS_FYO','.RData')))) {
return()
}
RUS_FYO <- read_sfn_data('RUS_FYO', folder)
site_metadata <- get_site_md(RUS_FYO)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- RUS_FYO %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT-3")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- RUS_FYO %>%
get_env_md() %>%
mutate(env_time_zone = "19UTC+03:00, C")
get_timestamp(RUS_FYO) <- good_timestamp
get_solar_timestamp(RUS_FYO) <- good_solar_timestamp
get_env_md(RUS_FYO) <- good_env_md
save(RUS_FYO, file = file.path(folder, 'RUS_FYO.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(RUS_FYO) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(RUS_FYO)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'RUS_FYO_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(RUS_FYO) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(RUS_FYO)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'RUS_FYO_sapf_flags.csv'
))
# env_data
get_env_data(RUS_FYO) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(RUS_FYO)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'RUS_FYO_env_data.csv'
))
# env_flags
get_env_flags(RUS_FYO) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(RUS_FYO)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'RUS_FYO_env_flags.csv'
))
# env_md
get_env_md(RUS_FYO) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'RUS_FYO_env_md.csv'
))
}
}
IDN_JAM_RUB_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('IDN_JAM_RUB','.RData')))) {
return()
}
IDN_JAM_RUB <- read_sfn_data('IDN_JAM_RUB', folder)
site_metadata <- get_site_md(IDN_JAM_RUB)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- IDN_JAM_RUB %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT-7")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- IDN_JAM_RUB %>%
get_env_md() %>%
mutate(env_time_zone = "28UTC+07:00, G")
get_timestamp(IDN_JAM_RUB) <- good_timestamp
get_solar_timestamp(IDN_JAM_RUB) <- good_solar_timestamp
get_env_md(IDN_JAM_RUB) <- good_env_md
save(IDN_JAM_RUB, file = file.path(folder, 'IDN_JAM_RUB.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(IDN_JAM_RUB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(IDN_JAM_RUB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_RUB_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(IDN_JAM_RUB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(IDN_JAM_RUB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_RUB_sapf_flags.csv'
))
# env_data
get_env_data(IDN_JAM_RUB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(IDN_JAM_RUB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_RUB_env_data.csv'
))
# env_flags
get_env_flags(IDN_JAM_RUB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(IDN_JAM_RUB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_RUB_env_flags.csv'
))
# env_md
get_env_md(IDN_JAM_RUB) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_RUB_env_md.csv'
))
}
}
IDN_JAM_OIL_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('IDN_JAM_OIL','.RData')))) {
return()
}
IDN_JAM_OIL <- read_sfn_data('IDN_JAM_OIL', folder)
site_metadata <- get_site_md(IDN_JAM_OIL)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- IDN_JAM_OIL %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT-7")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- IDN_JAM_OIL %>%
get_env_md() %>%
mutate(env_time_zone = "28UTC+07:00, G")
get_timestamp(IDN_JAM_OIL) <- good_timestamp
get_solar_timestamp(IDN_JAM_OIL) <- good_solar_timestamp
get_env_md(IDN_JAM_OIL) <- good_env_md
save(IDN_JAM_OIL, file = file.path(folder, 'IDN_JAM_OIL.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(IDN_JAM_OIL) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(IDN_JAM_OIL)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_OIL_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(IDN_JAM_OIL) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(IDN_JAM_OIL)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_OIL_sapf_flags.csv'
))
# env_data
get_env_data(IDN_JAM_OIL) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(IDN_JAM_OIL)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_OIL_env_data.csv'
))
# env_flags
get_env_flags(IDN_JAM_OIL) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(IDN_JAM_OIL)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_OIL_env_flags.csv'
))
# env_md
get_env_md(IDN_JAM_OIL) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'IDN_JAM_OIL_env_md.csv'
))
}
}
ESP_PRA_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('ESP_PRA','.RData')))) {
return()
}
ESP_PRA <- read_sfn_data('ESP_PRA', folder)
site_metadata <- get_site_md(ESP_PRA)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- ESP_PRA %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT-1")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- ESP_PRA %>%
get_env_md() %>%
mutate(env_time_zone = "17UTC+01:00, A")
get_timestamp(ESP_PRA) <- good_timestamp
get_solar_timestamp(ESP_PRA) <- good_solar_timestamp
get_env_md(ESP_PRA) <- good_env_md
save(ESP_PRA, file = file.path(folder, 'ESP_PRA.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(ESP_PRA) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_PRA)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_PRA_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(ESP_PRA) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_PRA)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_PRA_sapf_flags.csv'
))
# env_data
get_env_data(ESP_PRA) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_PRA)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_PRA_env_data.csv'
))
# env_flags
get_env_flags(ESP_PRA) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_PRA)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_PRA_env_flags.csv'
))
# env_md
get_env_md(ESP_PRA) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_PRA_env_md.csv'
))
}
}
ESP_LAS_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('ESP_LAS','.RData')))) {
return()
}
ESP_LAS <- read_sfn_data('ESP_LAS', folder)
site_metadata <- get_site_md(ESP_LAS)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- ESP_LAS %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT+0")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- ESP_LAS %>%
get_env_md() %>%
mutate(env_time_zone = "16UTC±00:00, Z")
get_timestamp(ESP_LAS) <- good_timestamp
get_solar_timestamp(ESP_LAS) <- good_solar_timestamp
get_env_md(ESP_LAS) <- good_env_md
save(ESP_LAS, file = file.path(folder, 'ESP_LAS.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(ESP_LAS) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_LAS)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_LAS_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(ESP_LAS) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_LAS)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_LAS_sapf_flags.csv'
))
# env_data
get_env_data(ESP_LAS) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_LAS)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_LAS_env_data.csv'
))
# env_flags
get_env_flags(ESP_LAS) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(ESP_LAS)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_LAS_env_flags.csv'
))
# env_md
get_env_md(ESP_LAS) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'ESP_LAS_env_md.csv'
))
}
}
USA_WVF_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('USA_WVF','.RData')))) {
return()
}
USA_WVF <- read_sfn_data('USA_WVF', folder)
site_metadata <- get_site_md(USA_WVF)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- USA_WVF %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT+5")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- USA_WVF %>%
get_env_md() %>%
mutate(env_time_zone = "9UTC-05:00, R")
get_timestamp(USA_WVF) <- good_timestamp
get_solar_timestamp(USA_WVF) <- good_solar_timestamp
get_env_md(USA_WVF) <- good_env_md
save(USA_WVF, file = file.path(folder, 'USA_WVF.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(USA_WVF) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_WVF)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_WVF_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(USA_WVF) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_WVF)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_WVF_sapf_flags.csv'
))
# env_data
get_env_data(USA_WVF) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_WVF)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_WVF_env_data.csv'
))
# env_flags
get_env_flags(USA_WVF) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_WVF)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_WVF_env_flags.csv'
))
# env_md
get_env_md(USA_WVF) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_WVF_env_md.csv'
))
}
}
USA_TNB_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('USA_TNB','.RData')))) {
return()
}
USA_TNB <- read_sfn_data('USA_TNB', folder)
site_metadata <- get_site_md(USA_TNB)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- USA_TNB %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT+5")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- USA_TNB %>%
get_env_md() %>%
mutate(env_time_zone = "9UTC-05:00, R")
get_timestamp(USA_TNB) <- good_timestamp
get_solar_timestamp(USA_TNB) <- good_solar_timestamp
get_env_md(USA_TNB) <- good_env_md
save(USA_TNB, file = file.path(folder, 'USA_TNB.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(USA_TNB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNB_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(USA_TNB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNB_sapf_flags.csv'
))
# env_data
get_env_data(USA_TNB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNB_env_data.csv'
))
# env_flags
get_env_flags(USA_TNB) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNB)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNB_env_flags.csv'
))
# env_md
get_env_md(USA_TNB) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNB_env_md.csv'
))
}
}
USA_TNO_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('USA_TNO','.RData')))) {
return()
}
USA_TNO <- read_sfn_data('USA_TNO', folder)
site_metadata <- get_site_md(USA_TNO)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- USA_TNO %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT+5")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- USA_TNO %>%
get_env_md() %>%
mutate(env_time_zone = "9UTC-05:00, R")
get_timestamp(USA_TNO) <- good_timestamp
get_solar_timestamp(USA_TNO) <- good_solar_timestamp
get_env_md(USA_TNO) <- good_env_md
save(USA_TNO, file = file.path(folder, 'USA_TNO.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(USA_TNO) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNO)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNO_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(USA_TNO) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNO)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNO_sapf_flags.csv'
))
# env_data
get_env_data(USA_TNO) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNO)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNO_env_data.csv'
))
# env_flags
get_env_flags(USA_TNO) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNO)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNO_env_flags.csv'
))
# env_md
get_env_md(USA_TNO) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNO_env_md.csv'
))
}
}
USA_TNP_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('USA_TNP','.RData')))) {
return()
}
USA_TNP <- read_sfn_data('USA_TNP', folder)
site_metadata <- get_site_md(USA_TNP)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- USA_TNP %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT+5")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- USA_TNP %>%
get_env_md() %>%
mutate(env_time_zone = "9UTC-05:00, R")
get_timestamp(USA_TNP) <- good_timestamp
get_solar_timestamp(USA_TNP) <- good_solar_timestamp
get_env_md(USA_TNP) <- good_env_md
save(USA_TNP, file = file.path(folder, 'USA_TNP.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(USA_TNP) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNP)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNP_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(USA_TNP) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNP)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNP_sapf_flags.csv'
))
# env_data
get_env_data(USA_TNP) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNP)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNP_env_data.csv'
))
# env_flags
get_env_flags(USA_TNP) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNP)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNP_env_flags.csv'
))
# env_md
get_env_md(USA_TNP) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNP_env_md.csv'
))
}
}
USA_TNY_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('USA_TNY','.RData')))) {
return()
}
USA_TNY <- read_sfn_data('USA_TNY', folder)
site_metadata <- get_site_md(USA_TNY)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- USA_TNY %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT+5")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- USA_TNY %>%
get_env_md() %>%
mutate(env_time_zone = "9UTC-05:00, R")
get_timestamp(USA_TNY) <- good_timestamp
get_solar_timestamp(USA_TNY) <- good_solar_timestamp
get_env_md(USA_TNY) <- good_env_md
save(USA_TNY, file = file.path(folder, 'USA_TNY.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(USA_TNY) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNY)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNY_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(USA_TNY) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNY)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNY_sapf_flags.csv'
))
# env_data
get_env_data(USA_TNY) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNY)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNY_env_data.csv'
))
# env_flags
get_env_flags(USA_TNY) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_TNY)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNY_env_flags.csv'
))
# env_md
get_env_md(USA_TNY) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_TNY_env_md.csv'
))
}
}
USA_PAR_FER_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('USA_PAR_FER','.RData')))) {
return()
}
USA_PAR_FER <- read_sfn_data('USA_PAR_FER', folder)
site_metadata <- get_site_md(USA_PAR_FER)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- USA_PAR_FER %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT+5")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- USA_PAR_FER %>%
get_env_md() %>%
mutate(env_time_zone = "9UTC-05:00, R")
get_timestamp(USA_PAR_FER) <- good_timestamp
get_solar_timestamp(USA_PAR_FER) <- good_solar_timestamp
get_env_md(USA_PAR_FER) <- good_env_md
save(USA_PAR_FER, file = file.path(folder, 'USA_PAR_FER.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(USA_PAR_FER) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_PAR_FER)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_PAR_FER_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(USA_PAR_FER) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_PAR_FER)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_PAR_FER_sapf_flags.csv'
))
# env_data
get_env_data(USA_PAR_FER) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_PAR_FER)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_PAR_FER_env_data.csv'
))
# env_flags
get_env_flags(USA_PAR_FER) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_PAR_FER)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_PAR_FER_env_flags.csv'
))
# env_md
get_env_md(USA_PAR_FER) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_PAR_FER_env_md.csv'
))
}
}
USA_INM_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('USA_INM','.RData')))) {
return()
}
USA_INM <- read_sfn_data('USA_INM', folder)
site_metadata <- get_site_md(USA_INM)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- USA_INM %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT+5")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- USA_INM %>%
get_env_md() %>%
mutate(env_time_zone = "9UTC-05:00, R")
get_timestamp(USA_INM) <- good_timestamp
get_solar_timestamp(USA_INM) <- good_solar_timestamp
get_env_md(USA_INM) <- good_env_md
save(USA_INM, file = file.path(folder, 'USA_INM.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(USA_INM) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_INM)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_INM_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(USA_INM) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_INM)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_INM_sapf_flags.csv'
))
# env_data
get_env_data(USA_INM) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_INM)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_INM_env_data.csv'
))
# env_flags
get_env_flags(USA_INM) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(USA_INM)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_INM_env_flags.csv'
))
# env_md
get_env_md(USA_INM) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'USA_INM_env_md.csv'
))
}
}
COL_MAC_SAF_RAD_fixes <- function(folder, csv) {
if (!file.exists(file.path(folder, paste0('COL_MAC_SAF_RAD','.RData')))) {
return()
}
COL_MAC_SAF_RAD <- read_sfn_data('COL_MAC_SAF_RAD', folder)
site_metadata <- get_site_md(COL_MAC_SAF_RAD)
# we need to update both timestamps, and the environmental metadata
good_timestamp <- COL_MAC_SAF_RAD %>%
get_timestamp() %>%
lubridate::force_tz("Etc/GMT+5")
good_solar_timestamp <- good_timestamp %>%
solar_timestamp_creator(lat = site_metadata$si_lat, long = site_metadata$si_long)
good_env_md <- COL_MAC_SAF_RAD %>%
get_env_md() %>%
mutate(env_time_zone = "9UTC-05:00, R")
get_timestamp(COL_MAC_SAF_RAD) <- good_timestamp
get_solar_timestamp(COL_MAC_SAF_RAD) <- good_solar_timestamp
get_env_md(COL_MAC_SAF_RAD) <- good_env_md
save(COL_MAC_SAF_RAD, file = file.path(folder, 'COL_MAC_SAF_RAD.RData'))
# csv files
if (isTRUE(csv)) {
# sapf_data
get_sapf_data(COL_MAC_SAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(COL_MAC_SAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'COL_MAC_SAF_RAD_sapf_data.csv'
))
# sapf_flags
get_sapf_flags(COL_MAC_SAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(COL_MAC_SAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'COL_MAC_SAF_RAD_sapf_flags.csv'
))
# env_data
get_env_data(COL_MAC_SAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(COL_MAC_SAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'COL_MAC_SAF_RAD_env_data.csv'
))
# env_flags
get_env_flags(COL_MAC_SAF_RAD) %>%
dplyr::mutate(TIMESTAMP_solar = get_solar_timestamp(COL_MAC_SAF_RAD)) %>%
dplyr::select(dplyr::starts_with('TIMESTAMP'), everything()) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'COL_MAC_SAF_RAD_env_flags.csv'
))
# env_md
get_env_md(COL_MAC_SAF_RAD) %>%
readr::write_csv(path = file.path(
stringr::str_replace(folder, 'RData', 'csv'),
'COL_MAC_SAF_RAD_env_md.csv'
))
}
}
## MAIN FUNCTION ####
sfn_data_fixes <- function(db_folder, csv = FALSE) {
data_levels_folders <- file.path(db_folder, 'RData', c('plant', 'sapwood', 'leaf'))
# sites <- c('AUS_MAR_HSW_HIG', 'COL_MAC_SAF_RAD', 'ESP_YUN_T3_THI')
for (folder in data_levels_folders) {
AUS_MAR_HSW_HIG_fixes(folder, csv)
COL_MAC_SAF_RAD_fixes(folder, csv)
ESP_YUN_T3_THI_fixes(folder, csv)
USA_SMI_SER_fixes(folder, csv)
USA_SMI_SCB_fixes(folder, csv)
USA_CHE_ASP_fixes(folder, csv)
RUS_POG_VAR_fixes(folder, csv)
ZAF_SOU_SOU_fixes(folder, csv)
ZAF_RAD_fixes(folder, csv)
ZAF_FRA_FRA_fixes(folder, csv)
RUS_FYO_fixes(folder, csv)
IDN_JAM_RUB_fixes(folder, csv)
IDN_JAM_OIL_fixes(folder, csv)
ESP_PRA_fixes(folder, csv)
ESP_LAS_fixes(folder, csv)
USA_WVF_fixes(folder, csv)
USA_TNB_fixes(folder, csv)
USA_TNO_fixes(folder, csv)
USA_TNP_fixes(folder, csv)
USA_TNY_fixes(folder, csv)
USA_PAR_FER_fixes(folder, csv)
USA_INM_fixes(folder, csv)
COL_MAC_SAF_RAD_fixes(folder, csv)
sapfluxnetr:::.write_metadata_cache(file.path(folder))
}
}
## Execute fixes ####
# CHANGE folder VALUE WITH THE PATH TO THE DATABASE VERSION (0.1.3)
# CHANGE csv VALUE TO FALSE IF YOU DONT WANT TO FIX THE CSV FILES
sfn_data_fixes(db_folder = '.', csv = TRUE)
@MalditoBarbudo

This comment has been minimized.

Copy link
Owner Author

@MalditoBarbudo MalditoBarbudo commented Apr 19, 2019

This script can fix the known data bugs (sapfluxnet/sapfluxnetr#14) in SAPFLUXNET data.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.