lxHive Mongo Admin

01-distinct.statements.js

// get a list of all verb ids
db.getCollection('statements').distinct('statement.verb.id');
// get a list of all object activity ids
db.getCollection('statements').distinct( "statement.object.id");
// get a list of object definition extension sub property values
db.getCollection('statements').distinct( "statement.object.definition.extensions.https://lxhive[dot].com/recipes/extensions/object.subType");
// get a list of object id's containing "lesson"
db.getCollection('statements').find({"statement.object.id": /lesson/});
// get a list of all verb display["en-US"] values
db.getCollection('statements').distinct('statement.verb.display.en-US');

// get a  distinct list of verb ids with count for each id
db.getCollection('statements').aggregate([
    {
        $match: {
            'statement.verb.id': { $not: {$size: 0} }
        }
    },
    { $unwind: "$statement.verb.id" },
    {
        $group: {
            _id: {$toLower: '$statement.verb.id'},
            count: { $sum: 1 }
        }
    },
    { $sort : { count : -1} },
    { $limit : 100 }
]);

01-queries.js

//// get last 5 statements
db.getCollection('statements').find({}).limit(5).sort({"$natural": -1});

//// count statements having a specific context extension
db.getCollection('statements').find({'statement.context.extensions.https://lxhive[dot]com/recipes/extensions/example-extension': {$exists: 1}}).count()

02-admin-update-key-and-secret.js

db.getCollection('basicTokens').update(
    // query 
    {
        "_id" : ObjectId("<DOCUMENT ID>")
    },
    
    // update 
    {
        "$set": {
            "key": "<STRING>",
            "secret": "<STRING>"
        }
        
    },
    
    // options 
    {
        "multi" : false,  // update only one document 
        "upsert" : false  // insert a new document, if no existing document match the query 
    }
);

02-csv-export-example.js

/**
 * USAGE
 * save as `csv-export.js`
 * 
 * $ mongo --quiet localhost:27017/lxhive ./csv-export.js > dump.csv
 * i.e: mongo --quiet <connection>/<database> ./csv-export.js > <path/to/file>.csv
 *
 * altenatively edit the commented lines below and start without connection arg
 * $ mongo --quiet ./csv-export.js > dump.csv
 */

//conn = new Mongo();
//db = conn.getDB('<db>');

var objType = function(val) {
    return (typeof val.objectType !== 'undefined') ? val.objectType : 'Activity';
};

var ifi = function(val) {
    if(typeof val.mbox !== 'undefined') {
        return val.mbox;
    }

    if(typeof val.mbox_sha1sum !== 'undefined') {
        return val.mbox_sha1sum;
    }

    if(typeof val.openid !== 'undefined') {
        return val.openid;
    }

    if(typeof val.account !== 'undefined') {
        return val.account;
    }

    return 'undefined';
};

var agentHuman = function(val) {
    var name = val.name || '';
    if (name) {
        return name;
    }

    return ifi(val);
};

var activityHuman = function(val) {
    var obj = val.definition || {};
    obj = obj.name || {};

    for (var key in obj) {
        if(obj[key]) {
            return obj[key];
        }
    }

    return val.id;
};

var objHuman = function(val) {

    var type = objType(val);

    if (type === 'Activity') {
        return activityHuman(val);
    }

    if (type === 'Agent') {
        return agentHuman(val);
    }

    if (type === 'Group') {
        return agentHuman(val);
    }

    if (type === 'StatementRef') {
        return val.id;
    }

    if (type === 'StatementRef') {
        return val;
    }

    return val;
};

var verbHuman = function(val) {
    var obj = val.display || {};

    for(var key in obj) {
        if(obj[key]) {
            return obj[key];
        }
    }

    return val.id;
};

var sanitize = function(val) {
    var  type = typeof val;
    if ( val === null || type === 'number' || type === 'boolean') {
        return '"' + val + '"';
    }
    if (type !== 'string') {
        return JSON.stringify(val);
    }
    return '"' + val.replace('"', '\'') + '"';
};

// 1. header
print([
    "#",
    "id",
    "stored",
    "actor",
    "verb",
    "object",
].join(','));

var count = 0;
db.statements.find()
.sort({
    $natural: -1
})
.forEach(function(d){
    var smt =  d.statement;
    // printjson(smt);
    // return;

    var oName = objHuman(smt.object);
    var vName = verbHuman(smt.verb);
    var aName = agentHuman(smt.actor);
    
    // 2. values
    var res = [
        sanitize(count),
        sanitize(smt.id),
        sanitize(smt.stored),
        sanitize(aName),
        sanitize(vName),
        sanitize(oName),
        //sanitize(smt.verb.id),
        //sanitize(smt.object.id || '')
    ].join(',');

    print(res);
    count++;
});

03-admin-update-permissions.js

db.getCollection('basicTokens').update(
    // query 
    {
        //"name" : "joerg"
        "_id" : ObjectId("<DOCUMENT ID>")
    },
    
    // update 
    {
        "$set": {
            "permissions" : [ 
                "super", 
                "statements/write", 
                "statements/read/mine", 
                "statements/read", 
                "state", 
                "define", 
                "profile", 
                "all/read", 
                "all", 
                "attachments", 
                "ext/extendedquery/statements"
            ],
        }
        
    },
    
    // options 
    {
        "multi" : false,  // update only one document 
        "upsert" : false  // insert a new document, if no existing document match the query 
    }
);

99-aggregate-verbIds-and-activityIds.js

db.getCollection('statements').aggregate([
    {
        // newest first
        $sort: {
            "_id": -1
        }
    },
    {
         $project: {
            /*statement: 1,*/
            
            //// statement, verb, object ids
            
            statementId: "$statement.id",
            verbId: "$statement.verb.id",
            activityId: "$statement.object.id",
            
            //// context activity id's
            
            // parent
            contextActivitiesParent: {
                $map: {
                     input: "$statement.context.contextActivities.parent",
                     as: "item",
                     in: {  $concat: ["", "$$item.id"] }
                }
            },
            // grouping
            contextActivitiesGrouping: {
                $map: {
                     input: "$statement.context.contextActivities.grouping",
                     as: "item",
                     in: {  $concat: ["", "$$item.id"] }
                }
            },
            // category
            contextActivitiesCategory: {
                $map: {
                     input: "$statement.context.contextActivities.category",
                     as: "item",
                     in: {  $concat: ["", "$$item.id"] }
                }
            },
            // other
            contextActivitiesOther: {
                $map: {
                     input: "$statement.context.contextActivities.other",
                     as: "item",
                     in: {  $concat: ["", "$$item.id"] }
                }
            }
         }
     }
]);

/**
 * output example
 */
 
// [{
//     "_id" : ObjectId("59ee87e2785e262fb577c2b7"),
//     "statementId" : "d427dbfd-58a1-0ab2-ac25-4dc41bb02c97",
//     "verbId" : "https://adlnet.gov/expapi/verbs/experienced",
//     "activityId" : "https://mysite.com/lessons/first-lesson",
//     "contextActivitiesParent" : [ 
//         "https://lxhive.com/recipes/wordpress/WP_Post/lessons", 
//         "https://lxhive.com/recipes/wordpress/WP_Taxonomy/tags"
//     ],
//     "contextActivitiesGrouping" : [ 
//         "https://lxhive.com/recipes/wordpress/type/lessons", 
//         "https://mysite.com/recipes/resource", 
//         "https://mysite.com/recipes/trigger", 
//         "https://lxhive.com/recipes/wordpress/plugin/lx-lessons"
//     ],
//     "contextActivitiesCategory" : [ 
//         "http://cumulus.brightcookie.com.au/tags/infographic"
//     ],
//     "contextActivitiesOther" : [ 
//         "https://mysite.com/wp-admin/admin-ajax.php"
//     ]
// }]

99-backup-database.md

• locate your mongo configuration file

locate mongod.conf
# ubuntu 16.x default install
# /etc/mongod.conf
# /etc/init/mongod.conf

redhat

# find / -name  mongod.conf
/etc/mongod.conf
/tmp/mongod.conf

• locate Mongo DB storage folder

grep -i dbpath /etc/mongod.conf
# dbPath: /var/lib/mongodb

• dump lxHive database into the storage folder

#### extra args
# --username= <NAME>
# --password=<PASSWORD>
# --port=3017
# --gzip
 cd /<path/to/lxHive> && mongodump --verbose --db=<DB NAME> --out=~/mongodump/$(date +%Y-%m-%d.%H:%M:%S)

• restore lxHive database into the storage folder

cd ~/mongodump/<date>
mongorestore --db <DB NAME> --verbose ./<FOLDER>

clear-collection.js

// clear a huge collection, disable timeouts
db.getCollection('logs').remove({}).maxTimeMS(0);

profiling.md

• https://docs.mongodb.com/manual/tutorial/manage-the-database-profiler/

db.system.profile.find()
// most recent 10 log entries
db.system.profile.find().limit(10).sort( { ts : -1 } ).pretty()

Profiling Levels

The following profiling levels are available:

0 - the profiler is off, does not collect any data. mongod always writes operations longer than the slowOpThresholdMs threshold to its log. This is the default profiler level. 1 - collects profiling data for slow operations only. By default slow operations are those slower than 100 milliseconds.

db.system.profile.find().limit(10).sort( { ts : -1 } ).pretty()

You can modify the threshold for “slow” operations with the slowOpThresholdMs runtime option or the setParameter command. See the Specify the Threshold for Slow Operations section for more information. 2 - collects profiling data for all database operations.

Set profiling level

• sets the profiling level for the current database to 0, which disables profiling
• sets the slow-operation threshold for the mongod instance to 20 milliseconds

db.setProfilingLevel(0,20);

Get profiling status

db.getProfilingStatus();

stats.js

/* db: megaBytes */

 db.stats(1024 * 1024)

/* single collection: megaBytes */

db.getCollection('connectionLogs').dataSize() / (1024 * 1024) // size value

db.getCollection('connectionLogs').stats(1024 * 1024) // stat values

/* collection stats */

conn = new Mongo();
db = conn.getDB('smsconnector');

const collections = db.getCollectionNames();

const stats = collections.map(collection => db[collection].stats());
const sortedStats = stats.sort((a, b) => b.size > a.size);

sortedStats.forEach((stat) => {
    printjson(stat);
});

systemd.md

Basic mongod service maintenance

Service status

# service status
systemctl status mongod.service

# service is active?
systemctl is-active mongod.service

# service in a failed state?
systemctl is-failed mongod.service

# service enabled and starts on boot?
systemctl is-enabled mongod.service

Inspect service configuration

# inspect service configuration
systemctl cat mongod.service

# list dependencies
systemctl list-dependencies mongod.service

Service control

sudo systemctl disable mongod.service
sudo systemctl enable mongod.service

sudo systemctl stop mongod.service
sudo systemctl start mongod.service
sudo systemctl restart mongod.service

sudo systemctl reload mongod.service
sudo systemctl reload-or-restart mongod.service

basic lxHive Mongo Cheatsheets