Littlefield Game Crawler (Supply Chain Management)

readme.md

Data fetcher for Littlefield game

When playing the Littlefield game in my supply chain management course I got fed up with how slow the interface was (and how long it took to get data out of it). I wrote a small data fetching script that stores the data in JSON for use in Excel, PowerBI, etc.

Setup

With NodeJS installed, copy the files (crawl_public.js and package.json) to a new folder. Then run npm install to install the dependencies. Also create a historical folder inside this folder.

Then go into crawl_public.js using your favorite text editor and enter your data from line 9 onwards. Institution, username and password are required. If you want to track cash balances of competitors, enter their usernames (found in the "overall standing" part of the game).

To run the crawler run: node crawl_public.js.

Note: if you know JavaScript you can add checks & notification code to the runAnalytics function. In technical terms: you can set up a cron job that runs the script every hour (I put it at 5 minutes after the hour) and notify you of data points exceeding certain thresholds in this function. I have excluded these here as I feel that would give you an unfair advantage.

PowerQuery for Excel/PowerBI

The JSON can be imported into Excel/PowerBI using the following PowerQuery code:

let
    Source = Json.Document(File.Contents(data.json")),
    data1 = Source[data],
    #"Converted to Table" = Table.FromList(data1, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
    #"Expanded Column1" = Table.ExpandListColumn(#"Converted to Table", "Column1"),
    #"Expanded Column2" = Table.ExpandRecordColumn(#"Expanded Column1", "Column1", {"name", "data"}, {"Column1.name", "Column1.data"}),
    #"Expanded Column1.data" = Table.ExpandRecordColumn(#"Expanded Column2", "Column1.data", Source[days]),
    #"Demoted Headers" = Table.DemoteHeaders(#"Expanded Column1.data"),
    #"Changed Type" = Table.TransformColumnTypes(#"Demoted Headers",{{"Column1", type text}, {"Column2", type number}, {"Column3", type number}, {"Column4", type number}, {"Column5", type number}, {"Column6", type number}, {"Column7", type number}, {"Column8", type number}, {"Column9", type number}, {"Column10", type number}, {"Column11", type number}, {"Column12", type number}, {"Column13", type number}, {"Column14", type number}, {"Column15", type number}, {"Column16", type number}, {"Column17", type number}, {"Column18", type number}, {"Column19", type number}, {"Column20", type number}, {"Column21", type number}, {"Column22", type number}, {"Column23", type number}, {"Column24", type number}, {"Column25", type number}, {"Column26", type number}, {"Column27", type number}, {"Column28", type number}, {"Column29", type number}, {"Column30", type number}, {"Column31", type number}, {"Column32", type number}, {"Column33", type number}, {"Column34", type number}, {"Column35", type number}, {"Column36", type number}, {"Column37", type number}, {"Column38", type number}, {"Column39", type number}, {"Column40", type number}, {"Column41", type number}, {"Column42", type number}, {"Column43", type number}, {"Column44", type number}, {"Column45", type number}, {"Column46", type number}, {"Column47", type number}, {"Column48", type number}, {"Column49", type number}, {"Column50", type number}, {"Column51", type number}, {"Column52", type number}, {"Column53", type number}, {"Column54", type number}, {"Column55", type number}, {"Column56", type number}, {"Column57", type number}, {"Column58", type number}, {"Column59", type number}, {"Column60", type number}, {"Column61", type number}, {"Column62", type number}, {"Column63", type number}, {"Column64", type number}, {"Column65", type number}, {"Column66", type number}, {"Column67", type number}, {"Column68", type number}, {"Column69", type number}}),
    #"Transposed Table" = Table.Transpose(#"Changed Type"),
    #"Promoted Headers" = Table.PromoteHeaders(#"Transposed Table", [PromoteAllScalars=true]),
    #"Changed Type1" = Table.TransformColumnTypes(#"Promoted Headers",{{"Column1.name", type number}, {"JobArrivals-data", Int64.Type}, {"JobWaiting-data", type number}, {"Inventory-data", Int64.Type}, {"Queue1-data", type number}, {"Util1-data", type number}, {"Queue2-data", type number}, {"Util2-data", type number}, {"Queue3-data", type number}, {"Util3-data", type number}, {"JobOut-contract1", Int64.Type}, {"JobOut-contract2", Int64.Type}, {"JobOut-contract3", Int64.Type}, {"LeadTime-contract1", type number}, {"LeadTime-contract2", type number}, {"LeadTime-contract3",type number}, {"Revenue-contract1", Int64.Type}, {"Revenue-contract2", Int64.Type}, {"Revenue-contract3", Int64.Type}}),
    #"Sorted Rows" = Table.Sort(#"Changed Type1",{{"Column1.name", Order.Ascending}}),
    #"Renamed Columns" = Table.RenameColumns(#"Sorted Rows",{{"Column1.name", "Day"}}),
    #"Changed Type2" = Table.TransformColumnTypes(#"Renamed Columns",{{"CashComp1-data", type number}, {"CashComp2-data", type number}, {"CashComp3-data", type number}, {"CashComp4-data", type number}, {"CashComp5-data", type number}, {"CashComp6-data", type number}, {"CashComp7-data", type number}, {"CashComp8-data", type number}, {"CashComp9-data", type number}, {"Cash-data", type number}}),
in
    #"Changed Type2"

Just change any CashCompX-data into the names of your competitors.

crawl_public.js

import got from "got";
import {CookieJar} from "tough-cookie";
import {writeFile} from 'fs/promises';

let cookieJar = new CookieJar();
let client = got.extend({ cookieJar });

// Enter details here
const settings = {
    insitution: "", // This is part of your login url (e.g. https://op.responsive.net/lt/lse/ --> lse)
    username: "",
    password: "",
    competitors: ["competitor-username1", "competitor-username2", "etc"] // You can add more competitors here
}
// ===================

let sessionUrl = "https://op.responsive.net/Littlefield/CheckAccess";
let authString = `institution=${settings.insitution}&ismobile=false&id=${settings.username}&password=${settings.password}`;

let dataRegex = "{label: '(.+?)'.+points: '(.+?)'}";

let dataUrls = {
    "Cash":         "https://op.responsive.net/Littlefield/Plot?data=CASH&x=all&team="+settings.username,
    "JobArrivals":  "https://op.responsive.net/Littlefield/Plot?data=JOBIN&x=all",
    "JobWaiting":   "https://op.responsive.net/Littlefield/Plot?data=JOBQ&x=all",
    "Inventory":    "https://op.responsive.net/Littlefield/Plot?data=INV&x=all",
    "Queue1":       "https://op.responsive.net/Littlefield/Plot?data=S1Q&x=all",
    "Util1":        "https://op.responsive.net/Littlefield/Plot?data=S1UTIL&x=all",
    "Queue2":       "https://op.responsive.net/Littlefield/Plot?data=S2Q&x=all",
    "Util2":        "https://op.responsive.net/Littlefield/Plot?data=S2UTIL&x=all",
    "Queue3":       "https://op.responsive.net/Littlefield/Plot?data=S3Q&x=all",
    "Util3":        "https://op.responsive.net/Littlefield/Plot?data=S3UTIL&x=all",
    "JobOut":       "https://op.responsive.net/Littlefield/Plot?data=JOBOUT&x=all",
    "LeadTime":     "https://op.responsive.net/Littlefield/Plot?data=JOBT&x=all",
    "Revenue":      "https://op.responsive.net/Littlefield/Plot?data=JOBREV&x=all"
}

// Add competitor data points
for(let x of settings.competitors){
    console.log(x);
    dataUrls[`Cash-${x}`] = "https://op.responsive.net/Littlefield/Plot?data=CASH&x=all&team="+x;
}

function spacedDataParser(spaceddata){
    let array = spaceddata.split(' ');
    let returnObject = {};
    if(array.length % 2 == 0){
        for (let i = 1; i < array.length; i = i+2) {
            returnObject[array[i-1]] = array[i];
        }
    }else{
        return false;
    }

    return returnObject;
}

async function getDataFromUrl(name, url) {
    let { body } = await client.get(url);
    let matches = [...body.matchAll(new RegExp(dataRegex,"g"))];
    let data = matches.map((x) => {
        return {
            name: name+"-"+x[1].replace(' ',''),
            data: spacedDataParser(x[2])
        }
    });

    return data;
}


async function runAnalytics(allData, lastDay) {
    let lastData = {};
    allData.forEach((metric) => {
        metric.forEach((point) => {
            lastData[point.name] = parseFloat(point.data[lastDay]);
        });
    });

    // Write your checks here (if you want)
    console.log(lastData);

    return;
}


async function Main(){

    // Authenticate
    await client.post(sessionUrl, {
        "headers": {
            "cache-control": "no-cache",
            "content-type": "application/x-www-form-urlencoded",
            "pragma": "no-cache",
            "Referer": sessionUrl,
        },
        "body": authString});
    
    // Get data
    let promises = [];
    for (const item in dataUrls) {
        let url = dataUrls[item];
        promises.push(getDataFromUrl(item, url));
    }

    let allData = [... await Promise.all(promises)];

    
    let uniquedays = [... new Set(
        allData.map((x) => {
            return x.map((y) => Object.keys(y.data));
        }).flat(3)
    )];

    let returnData = {
        days: uniquedays,
        data: allData
    }

    let lastDay = Math.floor(Math.max(...uniquedays.map((x) => parseFloat(x))));

    await writeFile('historical/day_'+lastDay+'_data.json', JSON.stringify(returnData));
    await writeFile('data.json', JSON.stringify(returnData));

    await runAnalytics(allData, lastDay);
}

Main();

package.json

{
  "name": "public-version",
  "version": "1.0.0",
  "description": "",
  "main": "crawl_public.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "got": "^12.0.1",
    "tough-cookie": "^4.0.0"
  }
}