A detailed look at the 2011 RIS Specification

dataset-ris.csv

dataset-types.csv

RIS-Specification.md

RIS Specification

This is to the best of my knowledge the RIS specification as it was defined in 2011 by the original author. See https://en.wikipedia.org/wiki/RIS_(file_format)#cite_note-:1-5.

(Re)producing the dataset

Requirements

In order to run the scripts used to produce the dataset the following programs must be available on your system path:

1. Docker
2. xsltproc
3. curl

Setup

chmod +x step1.sh
chmod +x step2.sh
./step1.sh

dataset-ris.csv

1. Open vendor/EN RIS Documentation.xls in Excel.

2. Then File > Save As… > Web Page (.htm) and save in the same directory!

3. Then run:

   ./step2.sh
   

dataset-types.csv

This is simply a copy/paste (sorted by field name) of the table on pages 8-9 of Direct Export RIS.pdf available in the vendor directory.

step1.sh

#!/bin/sh

rm -rfv *.zip
rm -rfv vendor
curl --output vendor.zip -L https://web.archive.org/web/20120526103719/http://refman.com/support/direct%20export.zip
unzip -d vendor vendor.zip

step2.sh

#!/bin/sh

JQ_DOCKER="docker run --rm -v $(pwd):/workspace -w /workspace dockerpinata/jq"

rm -rfv out
mkdir out
find vendor -name "sheet*.htm" | xargs -t -I{} sh -c 'xsltproc --html --output out/$(basename "{}" .htm).json transform-html-to-json.xsl "{}"'
$JQ_DOCKER sh -c 'find . -name "*.json" | xargs -t jq --raw-output --compact-output -f transform-extract.jq' | sort | tee out/all-tmp.json
$JQ_DOCKER jq --slurp '.' out/all-tmp.json | tee dataset-ris.json
$JQ_DOCKER jq --raw-output -f transform-csv.jq dataset-ris.json | tee dataset-ris.csv

transform-csv.jq

  (map(keys) | add | unique | . - ["TY", "ER"] | ["TY"] + .) as $cols
| map(. as $x | $cols | map($x[.] // "N/A"))
| [$cols] + .
| .[]
| @csv

transform-extract.jq

# e.g.,
#
# INPUT
#
# [ ["TY", "ABST"             ]
# , ["TY", "INPR"             ]
# , ["TY", "JFULL"            ]
# , ["TY", "JOUR"             ]
# , ["AB", "Abstract"         ]
# , ["AD", "Author Address"   ]
# , ["AN", "Accession Number" ]]
#
# OUTPUT
# 
# {"TY": "ABST" , "AB": "Abstract", "AD": "Author Address", "AN": "Accession Number"}
# {"TY": "INPR" , "AB": "Abstract", "AD": "Author Address", "AN": "Accession Number"}
# {"TY": "JFULL", "AB": "Abstract", "AD": "Author Address", "AN": "Accession Number"}
# {"TY": "JOUR" , "AB": "Abstract", "AD": "Author Address", "AN": "Accession Number"}
#

  map(select(.[0] != "TY")) as $fields
| map(select(.[0] == "TY"))
| map([.] + $fields)
| map(reduce .[] as $i ({}; . + {($i[0]): $i[1]}))
| map(.ER |= "(End of reference)")
| .[]

transform-html-to-json.xsl

<?xml version="1.0"?>

<xsl:stylesheet version="1.0"
                xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                xmlns:str="http://exslt.org/strings"
                xmlns:exsl="http://exslt.org/common"
                extension-element-prefixes="str exsl">

<xsl:output method="text" />

<xsl:template match="html">
[
  <!-- select <tr> with content -->
  <xsl:for-each select="body/table/tr[td[.!='']]">
    <xsl:apply-templates select="."/>
    <xsl:if test="position() != last()">,</xsl:if>
  </xsl:for-each>
]
</xsl:template>

<xsl:template match="tr">
["<xsl:apply-templates select="td[position()=1]"/>","<xsl:apply-templates select="td[position()=2]"/>"]
</xsl:template>

<!-- first <td> is the tag name -->
<xsl:template match="td[position()=1]">
  <xsl:variable name="t">
    <xsl:call-template name="normalise">
      <xsl:with-param name="str" select="."/>
    </xsl:call-template>
  </xsl:variable>
  <xsl:value-of select="substring($t, 1, 2)"/>
</xsl:template>

<xsl:template match="td[position()=2]">
  <xsl:call-template name="normalise">
    <xsl:with-param name="str" select="str:replace(., '`', '')"/>
  </xsl:call-template>
</xsl:template>

<!-- second <td> is the tag description -->
<xsl:template name="normalise">
  <xsl:param name="str"/>
  <xsl:variable name="out">
    <xsl:for-each select="str:tokenize($str, '&#x20;&#xA;&#xD;&#xC2;&#xA0;')">
      <xsl:value-of select="."/>
      <xsl:text> </xsl:text>
    </xsl:for-each>
  </xsl:variable>
  <xsl:value-of select="normalize-space($out)"/>
</xsl:template>

</xsl:stylesheet>