How to filter out bad CSV data with regular expressions in R

csv_regex_filter.R

library(dplyr)
library(stringr)

deaths<-read.csv("z_KoreanConflict.csv", header=TRUE, stringsAsFactors=FALSE)

# We want to make sure all data in INCIDENT_DATE is correct
# A quick investigation shows the data in a YYYYMMDD format, but some fields are empty
# A simple regular expression can just check for 8 digits
regEx = "^\\d{8}$"

# Here we can compare the total records in the data versus the records that match our regular expression
dim(deaths)  # The total number of records
sum(str_detect(deaths$INCIDENT_DATE, regEx)) # The number of records that match our regular expression

# Further investigation of the data shows the FATALITY_DATE sometimes contains the information we want
# Using a for loop we can replace INCIDENT_DATE with the value of FATALITY_DATE, but only if INCIDENT_DATE 
#  is NOT valid, but FATALITY_DATE is valid.
for (i in 1:dim(deaths)) {
  #if INCIDENT_DATE is bad, but FATALITY is good, replace it
  if (!str_detect(deaths$INCIDENT_DATE[i], regEx) & str_detect(deaths$FATALITY[i], regEx)) {
    #print(paste(i, "There is a mistake here"))
    deaths$INCIDENT_DATE[i]<-deaths$FATALITY[i]
  }
}

# Finally, remove any records that still do not have a valid INCIDENT_DATE value
deaths<-deaths %>%
  filter(str_detect(deaths$INCIDENT_DATE, regEx) == TRUE)

z_KoreanConflict.csv