nicd_hospital_surveillance_scrape

nicd_data_scrape.R

library(pdftools)
library(stringr)
library(rvest)

ExamplePDFLink = "https://www.nicd.ac.za/wp-content/uploads/2020/12/NICD-COVID-19-Daily-Sentinel-Hospital-Surveillance-report-National-20201228.pdf"

#Take url pdf file and extract table 2 (stats by province) and the report date
convertPDFToDataFrame<-function(PDFLink){
	HospitalData = pdf_text(PDFLink)
	JAM =HospitalData[2] #take second table
	JAM =  strsplit(JAM,"\n")
	JAM = trimws(JAM[[1]])#convert to rows
	
	#Find Hospital Status
	#This part hinges on the ordering of the "Admissions to date by discharge type" histogram to have "Discharged alive" first and "Died" second. Seems true for the considered date range October 2020-December 2021.
	HospStatus = c()
	IND = grep("Discharged alive ",JAM)
	IND2 = grep("Died ",JAM)[1]
	HospStatus$GeneralWard = as.double(JAM[IND+1])
	HospStatus$HighCare = as.double(head(strsplit(JAM[IND2+1]," ")[[1]],1))
	
	#Work on extracting table
	
	#to protect against different row order in table (e.g. https://www.nicd.ac.za/wp-content/uploads/2021/01/NICD-COVID-19-Daily-Sentinel-Hospital-Surveillance-report-National-20210127.pdf)
	NC_ix = grep("Northern Cape",JAM)
	LP_ix=grep("Limpopo",JAM)
	NW_ix=grep("North West",JAM)
	MP_ix=grep("Mpumalanga",JAM)
	FS_ix=grep("Free State",JAM)
	WC_ix=grep("Western Cape",JAM)
	EC_ix=grep("Eastern Cape",JAM)
	GP_ix=grep("Gauteng",JAM)
	KZN_ix=grep("KwaZulu-Natal",JAM)
    ix = c(NC_ix,LP_ix,NW_ix,MP_ix,FS_ix,WC_ix,EC_ix,GP_ix,KZN_ix)
	
	min_ix = min(ix[ix>30]) #to avoid announcement notices including province names.
	
	hosp_data_raw <- JAM[min_ix:grep("Total", JAM)] #get all data in the table
	hosp_data_raw <- str_split_fixed(hosp_data_raw, " {2,}", 10) #add columns based on double space and 10 columns
	hosp_data = data.frame(hosp_data_raw) #convert to dataframe
	colnames(hosp_data) = c("Province", "Facilities_Reporting", "Admissions_to_Date", "Died_to_Date", "Discharged_to_date", "Currently_Admitted", "Currently_in_ICU", "Currently_Ventilated", "Currently_Oxygenated", "Admissions_in_Previous_Day") #add column names
	
	#Row index for each Province
	NC_ix = grep("Northern Cape",hosp_data$Province)
	LP_ix=grep("Limpopo",hosp_data$Province)
	NW_ix=grep("North West",hosp_data$Province)
	MP_ix=grep("Mpumalanga",hosp_data$Province)
	FS_ix=grep("Free State",hosp_data$Province)
	WC_ix=grep("Western Cape",hosp_data$Province)
	EC_ix=grep("Eastern Cape",hosp_data$Province)
	GP_ix=grep("Gauteng",hosp_data$Province)
	KZN_ix=grep("KwaZulu-Natal",hosp_data$Province)
	
	#Fix empty spaces, which put last column out of position (in most datasets) for WC
	IND = which(hosp_data=="",arr.ind = TRUE)
	if(length(IND)>0){
		hosp_data[rbind(IND[2,])]=0
		IND[,2] = IND[,2]-1
		hosp_data[IND] = NA
	}
	
	ProvinceIndex = c(WC_ix,EC_ix,GP_ix,KZN_ix,LP_ix,FS_ix,MP_ix,NW_ix,NC_ix)
	
	#find date - many formats are used
	SearchText = HospitalData[1]
	FINDPOSDATEa = str_locate(SearchText,"/2020")
	FINDPOSDATEb = str_locate(SearchText,"/2021")
	FINDPOSDATEc = str_locate(SearchText," 2020")
	FINDPOSDATEd = str_locate(SearchText," 2021")
	if(sum(is.na(FINDPOSDATEa))==0){
		GETDATE = as.Date(trimws(substr(SearchText,1,FINDPOSDATEa[1,2])),format="%m/%d/%Y")
	}else if(sum(is.na(FINDPOSDATEb))==0){
		GETDATE = as.Date(trimws(substr(SearchText,1,FINDPOSDATEb[1,2])),format="%m/%d/%Y")
	}else if(sum(is.na(FINDPOSDATEc))==0){
		GETDATE = as.Date(trimws(substr(SearchText,1,FINDPOSDATEc[1,2])),format="%A, %d %B %Y")
	}else if(sum(is.na(FINDPOSDATEd))==0){
		if(FINDPOSDATEd[1,2]>32){
			GETDATE = as.Date(trimws(substr(SearchText,1,FINDPOSDATEd[1,2])),format="%A, %d %B %Y")
		}else{
			GETDATE = as.Date(trimws(substr(SearchText,1,FINDPOSDATEd[1,2])),format="%d %B %Y")
		}
	}else{
		GETDATE=NA
	}
	
	return(list(hosp_data=hosp_data, hosp_status = HospStatus, dateis=GETDATE, ProvinceIndex = ProvinceIndex))
}

#Using a list of links, extract data in format of https://github.com/dsfsi/covid19za/blob/master/data/nicd_hospital_surveillance_data.csv
GetDataROW <- function(LINKS){
LL = length(LINKS)

#initialise columns as NA
date=rep(NA,LL)
YYYYMMDD=rep(NA,LL)
total_admissions=rep(NA,LL)
WC = rep(NA,LL)
EC = rep(NA,LL)
GP = rep(NA,LL)
KZN = rep(NA,LL)
LP = rep(NA,LL)
FS = rep(NA,LL)
MP = rep(NA,LL)
NW = rep(NA,LL)
NC = rep(NA,LL)
current_num_in_hospital = rep(NA,LL)
general = rep(NA,LL)
high_care = rep(NA,LL)
ICU = rep(NA,LL)
isolation = rep(NA,LL)
total_healthcare_workers_admitted = rep(NA,LL)
num_discharged_alive = rep(NA,LL)
hospital_deaths = rep(NA,LL)
source = rep(NA,LL)

#for each dataset extract element values
for(aa in 1:LL){
print(aa)
	source[aa] = LINKS[aa]
	
	try({
	ALLTODAY = convertPDFToDataFrame(source[aa])
	TODAY = ALLTODAY$hosp_data
	date[aa] = as.character(ALLTODAY$dateis)
	YYYYMMDD[aa] = as.character(format(ALLTODAY$dateis,"%Y%m%d"))
	PROIndex = ALLTODAY$ProvinceIndex
	
	total_admissions[aa] = tail(TODAY$Admissions_to_Date,1)
	WC[aa] = TODAY$Admissions_to_Date[PROIndex[1]]
	EC[aa] = TODAY$Admissions_to_Date[PROIndex[2]]
	GP[aa] = TODAY$Admissions_to_Date[PROIndex[3]]
	KZN[aa] = TODAY$Admissions_to_Date[PROIndex[4]]
	LP[aa] = TODAY$Admissions_to_Date[PROIndex[5]]
	FS[aa] = TODAY$Admissions_to_Date[PROIndex[6]]
	MP[aa] = TODAY$Admissions_to_Date[PROIndex[7]]
	NW[aa] = TODAY$Admissions_to_Date[PROIndex[8]]
	NC[aa] = TODAY$Admissions_to_Date[PROIndex[9]]
	current_num_in_hospital[aa] = TODAY$Currently_Admitted[28]
	general[aa] = ALLTODAY$hosp_status$GeneralWard
	high_care[aa] = ALLTODAY$hosp_status$HighCare
	ICU[aa] = TODAY$Currently_in_ICU[28]
	isolation[aa] = NA #no longer collected, last date: 02-09-2020
	total_healthcare_workers_admitted[aa] = NA #no longer collected, last date 08-10-2020
	num_discharged_alive[aa] = TODAY$Discharged_to_date[28]
	hospital_deaths[aa] = TODAY$Died_to_Date[28]
	})	
}

#combine data	
	DATA = data.frame(date,YYYYMMDD,total_admissions,WC,EC,GP,KZN,LP,FS,MP,NW,NC,current_num_in_hospital,general,high_care,ICU,isolation,total_healthcare_workers_admitted,num_discharged_alive,hospital_deaths,source)
return(DATA)
}



#Link approach

    # 1.)  identify file locations from webpage
    webpages<-read_html("https://www.nicd.ac.za/diseases-a-z-index/disease-index-covid-19/surveillance-reports/daily-hospital-surveillance-datcov-report/")
	LinkBOXES = html_nodes(webpages,".elementor-button.elementor-size-xs")  #find tags of boxes
	LINKS = html_attr(LinkBOXES,"href")  #find url in these html elements
	USE_ORDER = rev(LINKS) #make chronological order
	#remove respiratory pathogen reports
	UU = grep("Pathogen",USE_ORDER) #index of links containing "Pathogen" in list of URLS (12 found as of December 2nd 2021)
	USE_ORDER = USE_ORDER[-UU] #remove these links
	
	#Note that prior to 27th October 2020 less than 10 columns. May want to fill this in manually.
	IND = grep("20201027",USE_ORDER)
	USE_ORDER= USE_ORDER[-c(1:(IND-1))]
	#This is all relevant links
	
	#2) extract tables
	TABLE = GetDataROW(USE_ORDER)
	
	write.csv(TABLE,"nicd_za_hospital_surveillance_data.csv",row.names=FALSE)

nicd_za_hospital_surveillance_data.csv

Updated to include patients in general and in high care.
Note that isolation was no longer reported after 02-09-2020; and admitted healthcare workers were no longer reported after 08-10-2020