MarkLogic document type inventorying script. This makes a list of the unique document types in the current database, and estimates how many of each of them there are.

documentInventory.js

/**
 * Document Type Inventory in MarkLogic v0.1
 * 
 * This version exists as a GitHub Gist at https://gist.github.com/masyukun/063f59a7f7d84eeaeeef2f23d05cad59
 * 
 * @license
 * Copyright (c) 2020 Matthew Royal
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

'use strict';

const inventoryDocuments = function() {

  // Output data structure
  let fileTypes = {};

  
  // Temporary data structures
  let fileExtTypes = {};
  let docTypes = {};
  let moreData = true;

  // List of XML type exclusions from previous iterations
  let exclusions = [];

  // Sample the URIs 3 times at 95% +/-5% confidence
  // efficiently approximates sampling at 99% +/-1% confidence
  let ii = 0;
  for (ii = 0; moreData && ii < 3; ++ii) {

    // Get a random sample of URIs at 95% +/-5% confidence
    let uris =
      fn.subsequence(
        cts.search( cts.andQuery([ exclusions ]), "score-random")
        , 1, 384
      ).toArray().map(function(d){

        // Get document URI and file extension
        let u = fn.baseUri(d)
        let p = u.toString().split("\.");
        let ext = p[p.length - 1];


        // XML files, ".xml" file extensions, and files with no extensions (assumed to be XML)
        if (d.documentFormat === "XML" || !ext || ext.toUpperCase() === "XML") {

          // Record the root element name and its namespace
          if (!docTypes[fn.nodeName(d.root)]) docTypes[fn.nodeName(d.root)] = fn.namespaceUriFromQName( fn.nodeName(d.root) );

          // Exclude this document type from later iterations
          // to help insure we find every unique document type
          exclusions.push(
            cts.notQuery(cts.elementQuery(fn.nodeName(d.root), cts.trueQuery()))
          );

        // Non-XML files and other file extensions
        } else {
          // Log the existence of each unique file extension
          if (!fileExtTypes[ext]) fileExtTypes[ext] = true;
        }
        
        return u;
      })
    
      // If there's nothing left to query, don't do any more iterations
      if (uris.constructor.name === "Array") { 
        if (uris.length <= 0) { break; }
      } else { 
        if (fn.count(uris) <= 0) { break; }
      }
  }


  // Estimate the number of each type of XML file
  for (let qname of Object.keys(docTypes)) {
    let namespace = (docTypes[qname] ? docTypes[qname] : "");
    let localname = (qname.split(":")[1] ? qname.split(":")[1] : qname);
    fileTypes[qname] = cts.estimate(cts.elementQuery(fn.QName(namespace, localname), cts.trueQuery()))
  }

  // Estimate the number of each type of non-XML file, by extension
  for (let e of Object.keys(fileExtTypes)) {
    fileTypes[e] = fn.count( cts.uriMatch("*."+e) );
  }
  
  // Return the JSON object
  return fileTypes
};