The R code to scrape NSIC data and code from PDF (for Nepal Standard Industrial Classification)

nsic-pdf-scraper.R

library(tm)

# download pdf from http://cbs.gov.np/image/data/Publication/Others/Nepal%20Standard%20Industrial%20Classification.pdf

# using tricks from http://stackoverflow.com/questions/3852354/extracting-text-data-from-pdf-files
nsic.pdf.file <- "data/nsic/Nepal Standard Industrial Classification.pdf"
Rpdf <- readPDF(control = list(text = "-layout"))
nsic.pdf.corpus <- VCorpus(URISource(nsic.pdf.file), readerControl = list(reader = Rpdf))
nsic.pdf.lines <- content(content(nsic.pdf.corpus)[[1]])

pattern.code.1 <- "^[ \r\n\t\f]*([A-Z]{1})[ ]+.*"
pattern.code.2 <- "^[ \r\n\t\f]*([0-9]{2})[ ]+.*"
pattern.code.3 <- "^[ \r\n\t\f]*([0-9]{3})[ ]+.*"
pattern.code.4 <- "^[ \r\n\t\f]*([0-9]{4})[ ]+.*"
pattern.desc <- "^[^\\(]*[\\(|-]([^\\)]+).*$"

start.flag = FALSE
start.desc.1.flag = FALSE
start.desc.4.flag = FALSE
df <- data.frame(categorycode=factor(),category=factor(),code3digit=factor(),code4digit=factor(),desc=factor(), stringsAsFactors=FALSE)
for(line in nsic.pdf.lines) {
  if(start.flag) {
    if(grepl(pattern.code.1, line)) {
      start.desc.1.flag = TRUE
      code.1 <- sub(pattern.code.1, "\\1", line[grepl(pattern.code.1, line)])
    }else if(grepl(pattern.code.2, line)) {
      # start.desc.flag = TRUE
      code.2 <- sub(pattern.code.2, "\\1", line[grepl(pattern.code.2, line)])
    }else if(grepl(pattern.code.3, line)) {
      # start.desc.flag = TRUE
      code.3 <- sub(pattern.code.3, "\\1", line[grepl(pattern.code.3, line)])
    }else if(grepl(pattern.code.4, line)) {
      start.desc.4.flag = TRUE
      code.4 <- sub(pattern.code.4, "\\1", line[grepl(pattern.code.4, line)])
    }else if((start.desc.1.flag | start.desc.4.flag) & grepl(pattern.desc, line)) {
      if(start.desc.1.flag) {
        category <- sub(pattern.desc, "\\1", line[grepl(pattern.desc, line)])
        start.desc.1.flag = FALSE
      }
      if(start.desc.4.flag) {
        desc <- sub(pattern.desc, "\\1", line[grepl(pattern.desc, line)])
        start.desc.4.flag = FALSE
        df <- rbind(df, data.frame(categorycode=c(code.1), category=c(trimws(category)), code3digit=c(code.2), code4digit=c(code.4), desc=c(trimws(desc)),stringsAsFactors=FALSE), stringsAsFactors = FALSE)
      }
    }
  }
  if(grepl("Code number[ ]*Name of Industry / Description", line)) {
    start.flag = TRUE
    #start process once this line is found
  }
}

# using dataframe viewer to see the text not correctly grepped
# finding incorrectly grepped text, containing ? and { to find the codes to replace the text
for(digit in df[grep("\\?", df$desc),]$code4digit) {
  print(digit)
}

for(digit in df[grep("\\{", df$desc),]$code4digit) {
  print(digit)
}

# fixing incorrectly grepped desc
df$category[df$categorycode == "B"] <- "Fishing"

df$desc[df$code4digit == "0130"] <- "Growing of crops combined with farming of animals (mixed farming)"
df$desc[df$code4digit == "0140"] <- "Agricultural, animal husbandry and breeding service activities (except veterinary activities"
df$desc[df$code4digit == "1551"] <- "Distilling, rectifying and blending of sprits, ethyl alcohol production from fermented metrials"
df$desc[df$code4digit == "2022"] <- "Manufacture of builders carpentry and joinary"
df$desc[df$code4digit == "2029"] <- "Manufacture of other products of wood cork, bamboo, straw and plaiting material"
df$desc[df$code4digit == "2102"] <- "Manufacture of corrugated paper and paperboard and containers of paper and paperboard "
df$desc[df$code4digit == "2212"] <- "Publishing of news paper, journal and periodicals"
df$desc[df$code4digit == "2230"] <- "Reproduction of recorded media"
df$desc[df$code4digit == "2411"] <- "Manufacture of basic chemical, except fertilizer and nitrozen compond"
df$desc[df$code4digit == "2412"] <- "Manufacture of fertilizer and nitrozen compond"
df$desc[df$code4digit == "2422"] <- "Manufacture of paints varnishes and similar coating printing ink and mastics product"
df$desc[df$code4digit == "2423"] <- "Manufacture of pharmaceuticals, medicinal chemicals and botanical products"
df$desc[df$code4digit == "2424"] <- "Manufacture of soap and detergents, cleaning and polishing preparation, perfumes and toilate preparation"
df$desc[df$code4digit == "2511"] <- "Manufacture of rubber tyres and tubes retreading and rebuilding of rubber tyres"
df$desc[df$code4digit == "2693"] <- "Manufacture of structural non refractory clay and ceramic products"
df$desc[df$code4digit == "2891"] <- "Foring, pressing, stamping and roll- forming of metal powder metallurgy"
df$desc[df$code4digit == "2892"] <- "Treatment and coataing of metals ; general mechanical engineering on a fee and contract basis"
df$desc[df$code4digit == "2911"] <- "Manufacture of engines and turbines, except aircraft, vehicle and cycles engines"
df$desc[df$code4digit == "2912"] <- "Manufacture of pump compressors taps and valves"
df$desc[df$code4digit == "2922"] <- "Manufacture of machine tools"
df$desc[df$code4digit == "2924"] <- "Manufacture of mechanery for mining, qurring and construction"
df$desc[df$code4digit == "3000"] <- "Manufacturer of office, accounting and computing machinery"
df$desc[df$code4digit == "3110"] <- "Manufacture of electric motors, generator and transformer"
df$desc[df$code4digit == "3190"] <- "Manufacture of other electrical equipment n.e.c."
df$desc[df$code4digit == "3220"] <- "Manufacture of radio and television transmitter and apparatus for line telephony and line telegraphy"
df$desc[df$code4digit == "3230"] <- "Manufacture of radio and television receivers, sound or video recording or re producing apparatus and associated goods"
df$desc[df$code4digit == "3312"] <- "Manufacture of instruments and appliance for measuring, checking, testing and other purpose except industrial process and control equipment"
df$desc[df$code4digit == "3330"] <- "Manufacture of watch and clocks"
df$desc[df$code4digit == "3720"] <- "Recycling of non -metal waste and scrap"
df$desc[df$code4digit == "7512"] <- "Regulation of the activities of agencies that provide health care, education cultural services and other social services and other social services excluding social security"
df$desc[df$code4digit == "7513"] <- "Regulation of and contribution to more efficient operation of business"
df$desc[df$code4digit == "2696"] <- "Cut stones"
df$desc[df$code4digit == "5211"] <- "Retail sale in non-specialized stores with food, beverages or tobacco predominating"
df$desc[df$code4digit == "6220"] <- "Non scheduled air transport"
df$desc[df$code4digit == "6711"] <- "Administration of financial market"
df$desc[df$code4digit == "2021"] <- "Manufacture of veneer sheets, plywoods lamin board and other panels and boards product"

write.csv(df, "data/nsic/codes.csv", row.names = FALSE)

# verifying results
library(dplyr)
df %>%
  group_by(categorycode) %>%
  count()

nsic_codes.csv