refresh mixpanel dropdowns

refreshMpDropdowns.js

  #!/usr/bin/env node

/*

THE WONDERFUL EVENT/PROP DROPDOWN RESURFACER
--------------------------------------------

purpose:
    mixpanel's dropdowns for events and props are great; but they only cache data that is from the last 30 days
    this sucks if you are importing data that's older than 30 days
    this script grabs an example of each unique event, in the time window you specify
    and then send a copy of each event to mixpanel with a time of NOW
    this effectively repopulates all the dropdowns with the events/props on your historical data


usage:
    1) modify the config to include your project secret (line 35); change to 'api-eu.mixpanel.com' (line 220) for EU residency
    2) run this script as:
        node refreshMpDropdowns.js

then see that all your dropdowns work again!


lovingly made for you by
  ___   _   __
 / _ \ | | / /
/ /_\ \| |/ / 
|  _  ||    \ 
| | | || |\  \
\_| |_/\_| \_/
              
ak@mixpanel.com              

*/

const config = {
	secret: `YOUR-PROJECT-SECRET`,
    projectId: `YOUR-PROJECT-ID`,
    //the range of dates to look for
    from: `2021-01-01`,
    to: `2021-10-01`
}




const http = require("https");
const qs = require("querystring");
global.RIGHT_NOW = Math.round(Date.now() / 1000);

//make a promise-aware version of fetch() using http.request
function fetch(params, postData) {
    return new Promise(function(resolve, reject) {
        const req = http.request(params, function(res) {
            // reject on bad status
            // if (res.statusCode < 200 || res.statusCode >= 300) {
            //     return reject(new Error('statusCode=' + res.statusCode));
            // }
            // cumulate data
            let body = [];
            res.on('data', function(chunk) {
                body.push(chunk);
            });
            // resolve on end
            res.on('end', function() {
                try {
                    body = JSON.parse(Buffer.concat(body).toString());
                } catch (e) {
                    reject(e);
                }
                resolve(body);
            });
        });
        // reject on request error
        req.on('error', function(err) {
            // This is not a "Second reject", just a different sort of failure
            reject(err);
        });
        if (postData) {
            req.write(postData);
        }
        // IMPORTANT
        req.end();
    });
}



async function main(config) {
    const auth = Buffer.from(config.secret + '::').toString('base64')

    //JQL FOR EVENT NAMES
    log(`calling /jql from ${config.from} to ${config.to}`)
    const jqlRes = await jqlQueryForEvents(config, auth);
    const allEvents = jqlRes.map(event => event.value);
    log(`   found ${allEvents.length} unique events`)

    //MOVE $TIME; KILL DISTINCT_ID
    log(`\nmodifying ${allEvents.length} events to update $time, $distinct_id, and wiping properties`)
    const processedEvents = makeCurrentAndWipeUID(allEvents);

    //get ride of $identify, $create_alias, and $merge
    const uniqueEvents = processedEvents.filter(ev => !['$identify', '$create_alias', '$merge'].includes(ev.event));
    log(`   removing $identify, $create_alias, and $merge events (${processedEvents.length - uniqueEvents.length} events)`)

    //BATCH THEM UP
    log(`\nwrapping ${uniqueEvents.length} events into ${Math.round(uniqueEvents.length/50 + 1)} batches`)
    const batchesOfEventsToSend = chunkArray(uniqueEvents, 150);

    //TRACK IT
    log(`\ncalling /import for project: ${config.projectId}`)
    for await (batch of batchesOfEventsToSend) {
        let sendToMp = await flushEventsToMixpanel(batch, auth)
        console.log(`       ${JSON.stringify(sendToMp)}`)

    }

    console.log(`\nALL FINISHED!\nin a few moments, all your project's dropdowns will be fixed! yay!`)

    process.exit()

}

main(config)


//UTILITIES
async function jqlQueryForEvents(config, auth) {
    const query = `function main() {
    return Events({
      from_date: '${config.from}',
      to_date:   '${config.to}'
    })
    .groupBy(["name"], mixpanel.reducer.any());
  }`

    const options = {
		hostname: 'mixpanel.com', // or 'eu.mixpanel.com'
        path: '/api/2.0/jql',
        method: 'POST',
        headers: {
            Accept: 'application/json',
            Authorization: `Basic ${auth}`,
            'Content-Type': 'application/x-www-form-urlencoded'
        }
    };
    const response = await fetch(options, qs.stringify({ script: query }))
    return response;
}

async function mpRawExport(config, eventName, auth) {
    const options = {
        hostname: 'data.mixpanel.com',
        path: `/api/2.0/export?from_date=${config.from}&to_date=${config.to}&limit=1&event=%5B%22${encodeURIComponent(eventName)}%22%5D`,
        method: 'GET',
        headers: {
            Accept: 'application/json',
            Authorization: `Basic ${auth}`,
            'Content-Type': 'application/x-www-form-urlencoded'
        }
    };
    const response = await fetch(options)
    return response;
}

function chunkArray(arr, size) {

    const arr2 = arr.slice(0),
        arrays = [];

    while (arr2.length > 0) {
        arrays.push(arr2.splice(0, size));
    }

    return arrays;
}

function makeCurrentAndWipeUID(arrayOfEvents) {
    for (mpEvent of arrayOfEvents) {
        //fix event name
        mpEvent.event = mpEvent.name


        //remove top level keys
        delete mpEvent.name;
        delete mpEvent.dataset;
        delete mpEvent.distinct_id;
        delete mpEvent.labels;
        delete mpEvent.sampling_factor;
        delete mpEvent.time;
        delete mpEvent.properties.$import
        delete mpEvent.properties.token
        delete mpEvent.properties.$mp_api_endpoint
        delete mpEvent.properties.$mp_api_timestamp_ms
        delete mpEvent.properties.mp_processing_time_ms
        delete mpEvent.properties.$insert_id
        delete mpEvent.properties.$distinct_id_before_identify
        delete mpEvent.properties.$is_reshuffled
        delete mpEvent.properties.mp_lib
        delete mpEvent.properties.$mp_api_timestamp_ms

        //delete all prop values so as not to pollute analysis
        for (let key in mpEvent.properties) {
            let emptyValue = getEmptyType(typeof mpEvent.properties[key])
            mpEvent.properties[key] = emptyValue
        }

        mpEvent.properties.distinct_id = `EVENT RESURFACER!`
        mpEvent.properties.$source = `EVENT RESURFACER!`
        mpEvent.properties.$insert_id = makeid(12)
        mpEvent.properties.time = RIGHT_NOW;        

    }

    return arrayOfEvents
}

async function flushEventsToMixpanel(batch, auth) {
    const options = {
        hostname: 'api.mixpanel.com', //or 'api-eu.mixpanel.com'
        path: `/import?strict=1`,
        method: 'POST',
        headers: {
            Accept: 'application/json',
            Authorization: `Basic ${auth}`,
            'Content-Type': 'application/json'

        }
    };


    const response = await fetch(options, JSON.stringify(batch))
    return response
}

function log(message) {
    console.log(message)
}

function sleep(milliseconds) {
    return new Promise((resolve) => setTimeout(resolve, milliseconds))
}

function getEmptyType(type = "") {
    switch (type) {
        case "undefined":
            return ``;
        case "boolean":
            return false;
        case "number":
            return 0;
        case "bigint":
            return 0;
        case "string":
            return ``;
        case "symbol":
            return ``;
        case "function":
            return {};
        case "object":
            if (Array.isArray(type)) {
                return []
            }
            else {
                return {};
            }
        default:
            return ``;
    }
}




function makeid(length) {
    var result = '';
    var characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
    var charactersLength = characters.length;
    for (var i = 0; i < length; i++) {
        result += characters.charAt(Math.floor(Math.random() *
            charactersLength));
    }
    return result;
}