A function to parse the data found on https://dev.w3.org/html5/html-author/charref into a usable reference/index table for R

charset_output.csv

parse_html_charsets.R

# System info -------------------------------------------------------------

# $ sysname       : chr "Linux"
# $ release       : chr "4.4.0-97-generic"
# $ version       : chr "#120-Ubuntu SMP Tue Sep 19 17:28:18 UTC 2017"
# $ platform      : chr "x86_64-pc-linux-gnu"
# $ arch          : chr "x86_64"
# $ os            : chr "linux-gnu"
# $ system        : chr "x86_64, linux-gnu"
# $ status        : chr ""
# $ major         : chr "3"
# $ minor         : chr "4.1"
# $ year          : chr "2017"
# $ month         : chr "06"
# $ day           : chr "30"
# $ svn rev       : chr "72865"
# $ language      : chr "R"
# $ version.string: chr "R version 3.4.1 (2017-06-30)"
# $ nickname      : chr "Single Candle"

data.parse_html_char_ref <- function(assign_as = NULL, ...){

  page <- xml2::read_html('https://dev.w3.org/html5/html-author/charref')
  blocks <- xml2::xml_find_all(page, './/tr')
  
  char_df <- plyr::ldply(blocks, function(i){
    # The overall groups/rgx/control-names etc, ie: the <tr> attributes not <td>
    top_data <- xml2::xml_attrs(i)
    # set names for binding and remove the data- pre-cursor used for js rendering
    r <- setNames(top_data, gsub('data-', '', names(top_data)))
    # get the <td>'s
    kids <- xml2::xml_children(i)
    # loop through, and assign the names of the values by the class attributes
    # contained in the html structure for the table-cell tags
    rr <- unlist(lapply(kids, function(j){
      setNames(stringi::stri_trim_both(
        xml2::xml_text(xml2::xml_contents(j))
      ), xml2::xml_attr(j, 'class'))  
    }), recursive = F)
    
    # bind into a flat row and coerce to a data.frame. For some reason this is 
    # the only way to return a 1d vectorized value which is shaped into a 1-row 
    # df 
    rbind(c(r, rr)) %>% as.data.frame()
    
  }) %>% 
    # normalize the case of the human-readable descriptions so we can use
    # this table as a hash(index table) later when looking up codepoints or
    # converting... lower = easier = faster grep.
    # Additionally unescape the character(which on linux is visible as it would be 
    # on the web...) value so that we have the codepoint representation as well.
    # This will give us a data.frame that looks like the following(row-wise)
    # 'data.frame':	1 obs. of  9 variables:
    #  $ title             : chr "mathematical double-struck small z"
    #  $ rendered_character: chr "𝕫"
    #  $ icu_category_rgx  : chr "Ll"
    #  $ icu_block         : chr "Mathematical Alphanumeric Symbols"
    #  $ icu_set           : chr "mmlextra"
    #  $ html_char_named   : chr "&zopf;"
    #  $ hex_cp            : chr "&#x1D56B;"
    #  $ decimal_cp        : chr "&#120171;"
    #  $ unicode_cp        : chr "\\U0001d56b"
  mutate(
    desc = tolower(desc), 
    unicode_cp = stringi::stri_escape_unicode(character)
    ) %>% 
    select(title = desc, 
           rendered_character = character, 
           icu_category_rgx = category,
           icu_block = block,
           icu_set = set,
           html_char_named = named, 
           hex_cp = hex, 
           decimal_cp = dec,
           unicode_cp)
  
  # This is a habit I've gotten into which simply assigns the output
  # into the global environment with an object name passed as an argument.
  # 
  # Works the same as would var_chars <- data.parse_html_char_ref()
  if(!is.null(assign_as)){
    assign(assign_as, char_df, envir = .GlobalEnv)
  }else {
    return(char_df)
  }
}