Skip to content

Instantly share code, notes, and snippets.

@ayushprd
Created July 24, 2020 08:33
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ayushprd/44f2ebf9ec677e1ce618b15f32c23d12 to your computer and use it in GitHub Desktop.
Save ayushprd/44f2ebf9ec677e1ce618b15f32c23d12 to your computer and use it in GitHub Desktop.
existing_data <- inputs %>% select(start_date, end_date, name) %>% filter(site_id==Y)
if (existing_data>1){
if (!is.null(process_data)){
# construct file names, will be used for querying the db
process_file = process_file.nc
raw_file = raw_file.nc
# check if processed file exists with overlapping date
if (nrow(process_res <- inputs %>% select(id, start_date, end_date, name) %>% filter(name==process_file && (end_date >= as.Date(req_start) && start_date <= as.Date(req_end))) == 1)){
actual_date_stage <- set_date_stage(process_res, req_start, req_end, process_data)
get_data <- FALSE
}
# if not check the raw file if it already exists for processing and overlaps
else if(nrow(raw_res <- inputs %>% select(id, start_date, end_date, name) %>% filter(name==raw_file && (end_date >= as.Date(req_start) && start_date <= as.Date(req_end))) == 1)){
actual_date_stage = set_date_stage(raw_res, req_start, req_end, get_data)
# if raw file exists extract its path
if (actual_date_stage$get_data == False){
existing_file <- dbfile.file(pass id and other info)
}
process_data <- TRUE
}
}
# if raw data/bands is requested check if there is any overlap
else if(nrow(raw_res <- inputs %>% select(id, start_date, end_date, name) %>% filter(name==raw_file && (end_date >= as.Date(req_start) && start_date <= as.Date(req_end))) == 1)){
actual_date_stage = set_date_stage(raw_res, req_start, req_end, get_data)
}
}
else{
if(!is.null(get_data){
get_data <- TRUE
}
if(!is.null(process_data){
process_data <- TRUE
}
}
output = remote_process(pass all arguments)
if(get_data){
dbfile.input.insert(output$raw_file)
}
if(process_data){
dbfile.input.insert(output$process_file)
}
# result - query df
# req_start, req_end - start, end requetsed by user
# stage - get_data or process_data
set_date_stage <- function(result, req_start, req_end, stage){
db_start = result$start_date # dates from existing db record
db_end = result$end_date
stage <- TRUE
if(db_start <= req_start && <= req_end <=db_end){
print("data already exits")
req_start <- NULL
req_end <- NULL
stage <- FALSE
# lower overlap case
}else if(req_start < db_start) && (db_start <= req_end && req_end <= db_end){
req_end <- db_start - 1
# upper overlap case
}else if (req_end > db_end) && (db_start <= req_start && req_start <= db_end){
req_start <- db_end + 1
}
return list(req_start, req_end, stage)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment