Sample code related to mongo using aggregation

mongoc.py

'''
# Sample api endpoint
@app.route('/api/v1/test')
def testapi():
    oid = oid_from_now(5)
    col = 'unit_test'
    result = []
    import re
    REGEX = re.compile('HDF-3.2.0.0*')
    query = {"$and": [{"_id": {"$gt": bson.ObjectId(oid)}},
             {"TX_STACK_BUILD_NR": {"$regex": REGEX}}]}
    projection = "TX_JOB_NAME"
    r = execute_distinct(col, query, projection)
    for j in r:
        # find one job for metadata
        job_data = {}
        pipeline = []
        query = {'job_name': j}
        projection = {'job_name': 1, 'job_state': 1}
        job_meta = execute_findone('job_state', query, projection=projection)
        job_data['job'] = j
        job_data['meta'] = job_meta
        pipeline.append({'$match': {'TX_JOB_NAME': j}})
        pipeline.append({
            '$group': {
                '_id': '$TX_COMPONENT',
                'tests': {
                    '$push': {
                        'os': '$TX_OS',
                        'class_name': '$class_name',
                        'test_name': '$test_name',
                        'status': '$status'
                    }
                }
            }
        })
        pipeline.append({
            '$sort': SON([('_id', -1)])
        })
        job_data['result'] = execute_aggregation(col, pipeline, True)
        result.append(job_data)
    return jsonify(results=result)

'''

'''
pipelines = []
pipelines.append({'$match': {'TX_STACK_VERSION': stack}})
pipelines.append(
    {'$group': {
        '_id': {'stack': '$TX_STACK_VERSION', 'job': '$TX_JOB_NAME'},
        'items':
            {
            '$push':
            {
                'os': '$TX_OS',
                'class_name': '$class_name',
                'test_name': '$test_name',
                'status': '$status',
                'job': '$TX_JOB_NAME'
            }
            }
        }
    })
pipelines.append({'$sort': {'_uid': -1}})
pipelines.append({'$project':
                    {
                    '_id': 0.0, 'stack': '$_id', 'items': '$items'
                    }
                    })
pipelines.append({'$limit': 1000})
try:
    for rel in col.aggregate(pipelines):
        result.append(rel)
except PyMongoError as e:
    logger.error('Code: {} Error: {}'.format(e.code, e.details))
finally:
    connection.close()
return jsonify(results=result)
'''


'''
pipeline = []
pipeline.append({'$match': {'TX_JOB_NAME': job}})
pipeline.append({
    '$group': {
        '_id': '$TX_COMPONENT',
        'tests': {
            '$push': {
                'os': '$TX_OS',
                'class_name': '$class_name',
                'test_name': '$test_name',
                'status': '$status'
            }
        }
    }
})
pipeline.append({
    '$sort': SON([('_id', -1)])
})
'''
'''
        pipeline.append({'$match': {'TX_JOB_NAME': j}})
        pipeline.append({
            '$group': {
                '_id': '$TX_COMPONENT',
                'tests': {
                    '$push': {
                        'os': '$TX_OS',
                        'class_name': '$class_name',
                        'test_name': '$test_name',
                        'status': '$status'
                    }
                }
            }
        })
        pipeline.append({
            '$sort': SON([('_id', -1)])
        })
'''

'''
# Find latst teststatus by component and status
# Hint taken from 
# https://stackoverflow.com/questions/22932364/mongodb-group-values-by-multiple-fields
# Requires pymongo 3.6.0+
from bson.son import SON
from pymongo import MongoClient

client = MongoClient("mongodb://host:port/")
database = client["test_results"]
collection = database["system_test"]

# Created with Studio 3T, the IDE for MongoDB - https://studio3t.com/

pipeline = [
    {
        u"$match": {
            u"TX_JOB_NAME": u"st-nifi-20180426-173435-011436-1"
        }
    }, 
    {
        u"$sort": SON([ (u"_id", 1), (u"TX_COMPONENT", 1) ])
    }, 
    {
        u"$group": {
            u"_id": {
                u"component": u"$TX_COMPONENT",
                u"class_name": u"$class_name",
                u"test_name": u"$test_name"
            },
            u"last": {
                u"$last": u"$$ROOT"
            }
        }
    }, 
    {
        u"$group": {
            u"_id": {
                u"component": u"$last.TX_COMPONENT",
                u"status": u"$last.status"
            },
            u"count": {
                u"$sum": 1.0
            }
        }
    }, 
    {
        u"$group": {
            u"_id": u"$_id.component",
            u"status": {
                u"$push": {
                    u"status": u"$_id.status",
                    u"total": u"$count"
                }
            },
            u"total": {
                u"$sum": u"$count"
            }
        }
    }
]

cursor = collection.aggregate(
    pipeline, 
    allowDiskUse = False
)
try:
    for doc in cursor:
        print(doc)
finally:
    client.close()
'''

'''
# Latest test-case status for components using mongo aggregation
# Requires pymongo 3.6.0+
from bson.son import SON
from pymongo import MongoClient

client = MongoClient("mongodb://host:port/")
database = client["test_results"]
collection = database["system_test"]

# Created with Studio 3T, the IDE for MongoDB - https://studio3t.com/

pipeline = [
    {
        u"$match": {
            u"TX_JOB_NAME": u"st-nifi-20180426-173435-011436-1"
        }
    }, 
    {
        u"$sort": SON([ (u"_id", 1), (u"TX_COMPONENT", 1) ])
    }, 
    {
        u"$group": {
            u"_id": {
                u"component": u"$TX_COMPONENT",
                u"class_name": u"$class_name",
                u"test_name": u"$test_name"
            },
            u"last": {
                u"$last": u"$$ROOT"
            }
        }
    }, 
    {
        u"$group": {
            u"_id": u"$_id.component",
            u"tests": {
                u"$push": {
                    u"os": u"$last.TX_OS",
                    u"class_name": u"$last.class_name",
                    u"test_name": u"$last.test_name",
                    u"status": u"$last.status"
                }
            },
            u"total": {
                u"$sum": 1.0
            }
        }
    }
]

cursor = collection.aggregate(
    pipeline, 
    allowDiskUse = False
)
try:
    for doc in cursor:
        print(doc)
finally:
    client.close()
'''

'''
# Computation on the client side
def get_latest_teststatus(collection, job, all=False):
    con = mongodb_conn()
    col = con[g_mongodb][collection]
    latest = []
    result = []
    total_pass = 0
    total_fail = 0
    query = {"TX_JOB_NAME": job}
    projection = {
                "TX_COMPONENT": 1,
                "os": 1,
                "class_name": 1,
                "test_name": 1,
                "status": 1
                }
    sort = [(u"_id", -1)]
    cursor = col.find(query, projection=projection, sort=sort)
    try:
        for doc in cursor:
            t = '{}{}{}'.format(
                        doc['TX_COMPONENT'],
                        doc["class_name"],
                        doc["test_name"]
                        )
            u = hashlib.md5(t.encode('utf-8')).hexdigest()
            if u not in latest:
                latest.append(u)
                doc['_id'] = doc['_id'].generation_time
                if all:
                    result.append(doc)
                if doc['status'] == "pass":
                    total_pass = total_pass + 1
                if doc['status'] == "fail":
                    total_fail = total_fail + 1
            else:
                print('JOB:{} Duplicate:{}'.format(job, doc))
    except PyMongoError as e:
        logger.error("Code: {} Error: {}".format(e.code, e.details))
    finally:
        con.close()
    return {"pass": total_pass, "fail": total_fail, "result": result}

'''