Fetches user data from GraphQL, formats it, sends to NRDB as queryable events.

users-to-nrql.js

var got = require('got');

/* -------------------CONFIGURATION-------------------------------------- */
var API_KEY = '<user_key>'; //add as a secure cred - Used to fetch data via GraphQL - preferably a master account key
var INGEST_KEY = '<ingest_key>'; //add as a secure cred - Used to insert data into NRDB
var ACCOUNT_ID = 1 //account to post events to
var EVENT_TYPE = 'NrUsers'; //eventType (table) that data is stored in
/* -------------------CONFIGURATION-------------------------------------- */

var GRAPH_API = 'https://api.newrelic.com/graphql';
var HEADERS = { 'Content-Type': 'application/json', 'Api-Key': API_KEY };

async function main() {
  var allUsers = [];
  var allGroups = [];

  let domains = await getAuthDomains();
  let accounts = domains.accounts;
  let authIds = domains.ad;

  let usersProms = [];
  let groupsProms = [];
  for (var d of authIds) {
    usersProms.push(getUsers(d.id, null, allUsers));
    groupsProms.push(getGroups(d.id, null, allGroups));
  }

  let usersData = await Promise.all(usersProms);
  let groupsData = await Promise.all(groupsProms);

  const map = new Map();
  usersData.forEach(item => map.set(item.authDomain, item));
  groupsData.forEach(item => map.set(item.authDomain, {...map.get(item.authDomain), ...item}));
  const mergedData = Array.from(map.values());

  let formatted = [];
  let aUser = null;

  for (var aSet of mergedData) {  // For each auth domain
    let authName = aSet.authDomain;
    aSet.users.map(usr => {  // For each user
      let userLower = null;
      let userAdded = false;
      let aUsersGroups = [];
      if (usr.email) {
        userLower = usr.email.toLowerCase();
      }
        if (usr.groups.groups.length == 0) {  // A user may have no groups
          console.warn(`User: ${usr.email} has no groups assigned in auth domain: ${authName}`)
        }
        usr.groups.groups.map(group => {  // For each group assigned to the user
          aUsersGroups.push(group.displayName)
          aSet.groups.map(gr => {  // For each defined group
            if ((gr.id == group.id) && (gr.roles.roles.length == 0)) {  // The group may either not exist or have no accounts assigned
              console.warn(`Group: ${gr.displayName} assigned to user: ${usr.email} has no accounts assigned in auth domain: ${authName}`)
            }
            gr.roles.roles.map(account => {  // For each account associated with the defined group
              accounts.map(acctId => {  // For each defined group account id
                if (gr.id == group.id) {  // If the defined group guid equals the assigned group guid
                  if (account.accountId == acctId.id) {  // If the defined group account id (not guid) equals the assigned group account id
                    if (usr.email) {  // If the user has an email address
                      if (!userLower.includes('@newrelic.com')) {  // Don't include newrelic addresses
                        aUser = { 'eventType': EVENT_TYPE, 'authDomain': authName, 'email': userLower, 'name': usr.name, 'userId': usr.id, 'lastActive': usr.lastActive, 'userGroups': aUsersGroups.toString(), 'billableType': usr.type.displayName, "acctId": account.accountId, "accountName": acctId.name };
                        formatted.push(aUser);
                        userAdded = true;
                      }
                    }
                  }
                }
              })
            })
          })
        })
        if (!userAdded && !userLower.includes('@newrelic.com')) {  // Add user if they have not been added above
          aUser = { 'eventType': EVENT_TYPE, 'authDomain': authName, 'email': userLower, 'name': usr.name, 'userId': usr.id, 'lastActive': usr.lastActive, 'billableType': usr.type.displayName, "acctId": null, "accountName": null };
          formatted.push(aUser);
        }
    })
  }

  let uniques = [...new Map(formatted.map(i => [JSON.stringify([i.email, i.acctId]), i])).values()];
  console.log(uniques.length);

  const resultsUniques = uniques.filter((hash => obj => !(hash.has(obj.email) || hash.add(obj.email) && false))(new Set));
  console.log("Number of unique emails:", resultsUniques.length);

  let chunked = await chunkData(uniques, 2000);
  for (var c=0; c<chunked.length; c++) {
    writeToNRDB(chunked[c]);
  }
}

function chunkData(d, size) {
  return new Promise((resolve, reject) => {
    let chunked = d.reduceRight((r,i,_,s) => (r.push(s.splice(0,size)),r),[]);
    resolve(chunked)
  })
}

async function writeToNRDB(payload) {
  var h = {
    'Content-Type': 'application/json',
    'X-Insert-Key': INGEST_KEY
    // 'Content-Encoding': 'gzip'
  };

  var opts = {
    url: `https://insights-collector.newrelic.com/v1/accounts/${ACCOUNT_ID.toString()}/events`,
    headers: h,
    json: payload
  };

  let resp = await got.post(opts);
  if (resp.statusCode == 200) {
    console.log('done');
  } else {
    console.log('Error posting to NRDB ' + resp.statusCode);
    console.log(resp.body);
    throw new Error('Failed to post to NRDB');
  }
}

async function getUsers(authDomainId, usersCursor, au) {
  var q = `
    query ($authDomainId: [ID!], $usersCursor: String) {
      actor {
        organization {
          userManagement {
            authenticationDomains(id: $authDomainId) {
              authenticationDomains {
                users(cursor: $usersCursor) {
                  users {
                    id
                    name
                    email
                    lastActive
                    type {
                      displayName
                      id
                    }
                    groups {
                      groups {
                        displayName
                        id
                      }
                    }
                  }
                  nextCursor
                }
                name
              }
              nextCursor
            }
          }
        }
      }
    }
  `;

  var opts = {
    url: GRAPH_API,
    headers: HEADERS,
    json: {
      'query': q,
      'variables': {
        "authDomainId": authDomainId,
        "usersCursor": usersCursor,
      }
    }
  };

  let resp = await got.post(opts).json();
  if (resp.errors) {
    console.log(JSON.stringify(resp.errors));
    throw new Error('Unable to get Users!');
  } else {
    let authName = resp.data.actor.organization.userManagement.authenticationDomains.authenticationDomains[0].name;
    let users = resp.data.actor.organization.userManagement.authenticationDomains.authenticationDomains[0].users.users;
    let nextUsersCursor = resp.data.actor.organization.userManagement.authenticationDomains.authenticationDomains[0].users.nextCursor;

    if (nextUsersCursor == null) {
      au = au.concat(users);
      return {'users': au, 'authDomain': authName };
    } else {
      au = au.concat(users);
      return getUsers(authDomainId, nextUsersCursor, au);
    }
  }
}

async function getGroups(authDomainId, groupsCursor, ag) {
  var q = `
    query ($authDomainId: [ID!], $groupsCursor: String) {
      actor {
        organization {
          authorizationManagement {
            authenticationDomains(id: $authDomainId) {
              authenticationDomains {
                groups(cursor: $groupsCursor) {
                  groups {
                    displayName
                    id
                    roles {
                      roles {
                        accountId
                        displayName
                        id
                        name
                        type
                      }
                      nextCursor
                    }
                  }
                  nextCursor
                }
                name
              }
            }
          }
        }
      }
    }
  `;

  var opts = {
    url: GRAPH_API,
    headers: HEADERS,
    json: {
      'query': q,
      'variables': {
        "authDomainId": authDomainId,
        "groupsCursor": groupsCursor
      }
    }
  };

  let resp = await got.post(opts).json();
  if (resp.errors) {
    console.log(JSON.stringify(resp.errors));
    throw new Error('Unable to get Groups!');
  } else {
    let authName = resp.data.actor.organization.authorizationManagement.authenticationDomains.authenticationDomains[0].name;
    let groups = resp.data.actor.organization.authorizationManagement.authenticationDomains.authenticationDomains[0].groups.groups;
    let nextGroupsCursor = resp.data.actor.organization.authorizationManagement.authenticationDomains.authenticationDomains[0].groups.nextCursor;

    if (nextGroupsCursor == null) {
      ag = ag.concat(groups);
      return {'groups': ag, 'authDomain': authName };
    } else {
      ag = ag.concat(groups);
      return getGroups(authDomainId, nextGroupsCursor, ag);
    }
  }
}

async function getAuthDomains() { //TODO: add pagination
  var q = `
  {
      actor {
        organization {
          accountManagement {
            managedAccounts {
              name
              id
            }
          }
          authorizationManagement {
            authenticationDomains {
              authenticationDomains {
                id
                name
              }
            }
          }
        }
      }
    }
  `;

  var opts = {
    url: GRAPH_API,
    headers: HEADERS,
    json: {'query': q, 'variables': {}}
  };

  let resp = await got.post(opts).json();
  if (resp.errors) {
    console.log(resp.errors);
    throw new Error('Unable to get Auth Domains!');
  } else {
    let ad = resp.data.actor.organization.authorizationManagement.authenticationDomains.authenticationDomains;
    let accounts = resp.data.actor.organization.accountManagement.managedAccounts;
    return {'ad': ad, 'accounts': accounts};
  }
}

main();