Quepid NDCG@10 developed by Bertrand

Bertrand NDCG@10

// Wrap the Quepid objects and API in a namespace.
// Simple pass-through required by our NDCG scorer.
let quepidApi = {};

(function(context) {

  context.getDocs = function() {
    return docs;
  }
  
  context.getBestDocs = function() {
    return bestDocs;
  }

  // Document's rating accessor
  context.docRating = function(position) {
    // console.log('Position ' + position + ' rating: ' + docRating(position));
    return docRating(position);
  }

  // Has a ranking position been rated?
  context.hasDocRating = function(position) {
    // console.log('Position ' + position + ' rating? ' + hasDocRating(position));
    return hasDocRating(position);
  }

})(quepidApi);

// Scorer's name space to avoid name collision with the Quepid-provided objects.
var NDCG_scorer = {};

(function(context) {

  let _quepidApi = null;

  // Logging function.
  function log(obj) {
    // Log messages on the developer's console.
    var debug = true
    if (debug === true) console.log(obj)
  }

  // --------------------------------------------------------------------
  // Retrieve a document's rating.
  // hasDocRating(): See mock above. 
  // docRating(): See mock above.
  // --------------------------------------------------------------------
  function ratingOrDefault(posn, defVal) {
    if (!_quepidApi.hasDocRating(posn)) {
      return defVal;
    }
    return _quepidApi.docRating(posn);
  }

  // --------------------------------------------------------------------
  // Log base 2.
  // --------------------------------------------------------------------
  function log2(num) {
    return Math.log(num) / Math.log(2);
  }

  // --------------------------------------------------------------------
  // Round with precision. Used for logging numbers.
  // --------------------------------------------------------------------
  function myRound(number, precision = 0) {
    const factor = Math.pow(10, precision);
    const tempNumber = number * factor;
    const roundedTempNumber = Math.round(tempNumber);
    return roundedTempNumber / factor;
  }

  // --------------------------------------------------------------------
  // Build the axtual list of ratings.
  // Arguments:
  // p: Max rank position (See 'p' in the Wikipedia article)
  // --------------------------------------------------------------------
  function actualRatings(rankPosition) {
    const ratings = [];
    for (let i = 0; i < rankPosition; i++) {
      const rating = ratingOrDefault(i, 0.0)
      // log("Doc " + i + ": " + rating)
      ratings.push(rating);
    }
    log("Actual Ratings: " + ratings);
    return ratings;
  }

  // --------------------------------------------------------------------
  // Build the ideal list of ratings.
  // Arguments:
  // p: Max rank position (See 'p' in the Wikipedia article)
  // --------------------------------------------------------------------
  function idealRatings(rankPosition) {
  	const idealRatings = [];
    const bestDocsCount = _quepidApi.getBestDocs().length;
    for (let i = 0; i < bestDocsCount && i < rankPosition; i++) {
    	idealRatings.push(_quepidApi.getBestDocs()[i].rating);
    }
    
    // Padd with 0 (Not rated) if necessary.
    for (let i = idealRatings.length; i < rankPosition; i++) {
      // log("Ideal ratings list: Padding with 0.0 at position (0-based) " + i);
      idealRatings.push(0.0);
    }
    log("Ideal ratings: " + idealRatings);
    return idealRatings;
  }

  // --------------------------------------------------------------------
  // DCG calculator
  // Arguments:
  // docList: List of rated documents.
  // p: Maximum rank position.
  // --------------------------------------------------------------------
  function dcg(ratings, p) {
    let dcgScore = 0
    for (var i = 1; i <= ratings.length && i <= p; i++) {
      const logPosition = log2(i + 1);
      const dcgAdder = ratings[i - 1] / logPosition
      dcgScore += dcgAdder
      log('i=' + i + " ratings[i-1]=" + ratings[i - 1] + "/log(i+1)=" + logPosition + " => DCG incr. of " + myRound(dcgAdder, 3) + " (DCG=" + myRound(dcgScore, 3) + ")")
    }
    log('DCG([' + ratings + ']) = ' + dcgScore)
    return dcgScore
  }

  // Public API: ndcg
  // Arguments:
  // 1. Ranking position
  // For example, for an NDCG@10, use p = 10
  //
  // ATTENTION: Due to current limitation in Quepid ("bestDocs" is
  // 10-deep at most), p must be <= max(Number of results shown, 10)
  // For example, for p = 10, set the "Number of Results to Show"
  // in the Settings to 10.  
  // 
  // 2. Rating scale max value: MUST MATCH THE MAX RATING SPECIFIED IN THE 
  // "Scale for query ratings" scorer configuration.
  context.ndcg = function(quepidApi, rankPosition, ratingScaleMax) {

    log(" RUNNING NDCG!!! ");
    log(" *************** ");
    
    _quepidApi = quepidApi;

    log('Docs count    : ' + _quepidApi.getDocs().length)
    log('bestDocs count: ' + _quepidApi.getBestDocs().length)

    // Log the objects for inspection in the developer's console
    log(_quepidApi.getDocs())
    log(_quepidApi.getBestDocs())

    log('Rank position (p)=' + String(rankPosition))
    log('Using scale 1-' + String(ratingScaleMax))

    var myDcg = dcg(actualRatings(rankPosition), rankPosition);
    var iDcg = dcg(idealRatings(rankPosition), rankPosition);

    // NDCG is ~ dcg(currDocs()) / dcg(bestDocs())
    var nDcg = myDcg / iDcg;

    log(" DCG: " + myDcg);
    log("iDCG: " + iDcg);
    log("NDCG: " + nDcg);

    return nDcg;
  }

})(NDCG_scorer);

setScore(NDCG_scorer.ndcg(quepidApi, 10, 10) * 100);