Generating client error reports

gistfile2.txt

npm install csv
mkdir csv
# manual work
# visit https://logstash.wikimedia.org/goto/1def4b2013f38bdcd40dfd51e831254c
# export formatted in " file uri split"
# copy the download to the csv folder

node index.js

index.js

const fs = require('fs');
const { title } = require('process');
const dir = fs.readdirSync('csv', {
    withFileTypes: true
});
const newRows = {};
let header;

let modified = [];

const lastRun = (function() {
    try {
        return JSON.parse(fs.readFileSync('out.json'));
    } catch (e) {
        return {}
    }
}());

function normalizeUrl(url) {
    url = url.replace('&action=raw&ctype=text/javascript', '');
    var urlMap = {
        'https://ko.wikipedia.org/w/index.php?oldid=2906886': 'https://ko.wikipedia.org/w/index.php?title=%EC%82%AC%EC%9A%A9%EC%9E%90:-revi/vector.js'
    };
    var prefix = url.split('/w/load.php')[0] + '/w/index.php?title=';
    if (url.indexOf('ext.globalCssJs.user') > -1) {
        var amps = url.split('&');
        var user = amps.filter((a) => a.indexOf('user=') > -1)[0].split('=')[1];
        return `${prefix}User:${user}/global.js`;
    }
    if (url.indexOf('ext.gadget.') > -1 ) {
        const title = url.split('ext.gadget.')[1].split('&')[0];
        return url + `&title=gadget:${title}&debug=true&only=scripts`;
    }
    if(urlMap[url]) {
        return urlMap[url];
    }
    return url;
}

function extractTitleFromUrl(url) {
    var titleSplit = url.split('title=');
    if(!titleSplit[1]) {
        titleSplit = url.split('oldid=');
    }
    return titleSplit[1].split('&')[0];
}

function isGadget(title) {
    return [
        'gadget',
        'mediawiki'
    ].filter((m) => title.toLowerCase().indexOf(m) > -1).length;
}

function gadgetAmbassador(url) {
    let ambassador;
    // Use https://sr.wikipedia.org/w/index.php?title=MediaWiki:Gadgets-definition&action=history
    const AMBASSADORS = {
        'https://en.wikipedia.org/w/index.php?title=MediaWiki:Gadget-Navigation_popups': 'Amorymeltzer',
        'commons.wikimedia.org': 'Lupo',
        'sr.wikipedia.org': 'Aca',
        'vec.wikipedia.org': 'Fierodelveneto',
        'ar.wikipedia.org': 'Omar2040',
        'pl.wikipedia.org': 'Wargo',
        'hu.wikipedia.org': 'Tacsipacsi',
        'ro.wikipedia.org': 'Strainu',
        'fr.wikipedia.org': 'Od1n',
        'hr.wikipedia.org': 'Kubura',
        'MediaWiki:Gadget-importUtility/proj.js': 'Holmium',
        //'en.wikipedia.org': 'MusikAnimal',
        'as.wikipedia.org': 'দিব্য দত্ত',
        'zh.wikipedia.org': 'Xiplus',
        'wikidata.org': 'Lucas Werkmeister',
        'tr.wikipedia.org/w/index.php?title=MediaWiki:Gadget-twinkle': 'Vito Genovese',


        'Gadget-twinkleprod.js': 'Amorymeltzer',
        'pnb.wikipedia.org/w/index.php?title=MediaWiki:Gadget-Numeral_converter.js': 'Abbas dhothar',
        'Gadget-Merge.js':'Mahir256',
        'MediaWiki:OSM.js': '-Kolossos',
        'Gadget-Twinkle.js': 'Amorymeltzer',
        'Gadget-DotsSyntaxHighlighter.js': 'Remember the dot',
        'MediaWiki:Gadget-popups': 'Amorymeltzer',
        'MediaWiki:Wikiminiatlas': 'Dschwen',
        'MediaWiki:Gadget-infoboxExport.js': 'Putnik'
    };
    Object.keys(AMBASSADORS).forEach((project) => {
        if (url.indexOf(project) > -1) {
            ambassador = AMBASSADORS[project];
        }
    });
    if(!ambassador) {
        console.log('Please add ambassador for', url);
    }
    return ambassador;
}

function isSupportedUrl(url) {
    return url && (
        url.indexOf('ext.gadget.allinterwiki') === -1  &&
        url.indexOf('ext.gadget') > -1 || (
            url.indexOf('title=') > -1 &&
            url.indexOf('> scriptElement') === -1 &&
            url.indexOf('a1728df2bf02accc67fd0806f81d2d9f/e7473eedfdec9c939143160fb6750fd5') === -1
        )
    );
}

function isUserScript(url) {
    if(url.indexOf('/') > -1 && (
        url.indexOf('User:') > -1 ||
        url.indexOf('Участник:') > -1
    )) {
        return true;
    }
    return false;
}

function extractUserNameFromTitle(title, url) {
    const t = title.split(':')[1];
    if(!isUserScript(title) && isGadget(title)) {
        return gadgetAmbassador(url);
    } else {
        return t ? t.split('/')[0] : t;
    }
}

dir.forEach((file) => {
    const f = file.name;
    modified.push(fs.lstatSync(`csv/${file.name}`).birthtimeMs);
    const lines = fs.readFileSync('csv/' + f).toString().split('\n');
    header = lines[0];
    lines.slice(1).forEach((line) => {
        let cols = line.replace(/\r/,'').split('",').map((c) => c.replace(/['"\,]/g, ''));
        if(!line) return;
        if(cols.length > 2) {
            cols = [ cols[0], cols.slice(1, cols.length - 1).join('/'), cols[cols.length - 1]];
        }

        if (isNaN( parseInt(cols[2], 10) )) {
           console.log(cols)
            throw new Error('Something went wrong' + cols[2], line);
        } else {
            cols[2] =  parseInt(cols[2], 10);
        }
        const url = normalizeUrl(cols[0].replace(/"/g, ''));
        if(!isSupportedUrl(url)) {
            return;
        }
        // change URL
        cols[0] = url;
        const title = extractTitleFromUrl(url);
        let username = extractUserNameFromTitle(title, url);
        if (username) {
            username = `@[[User:${username}]]`;
        }
        if (!newRows[title]) {
            // if different file modification time true, otherwise false.
            let repeatEntry;
            // are the files being parsed different from the last files?
            if(!lastRun.rows[title]) {
                repeatEntry = false;
            } else if (modified.join(',') !== lastRun.modified.join(',')) {
                repeatEntry = lastRun.rows[title] ? true : false;
            } else {
                repeatEntry = lastRun.rows[title][4];
            }

            newRows[title] = cols.concat(
                [ username, repeatEntry, 1 ]
            );
            if(newRows[title].length > 6 ){
                throw new Error('error state1');
            } 
        } else {
            //newRows[title][1] += cols[1];
            //console.log(newRows[title][2],cols[2]);
            newRows[title][1] += "/ " + cols[1];
            newRows[title][2] += cols[2];
            newRows[title][5] += 1;
            if(newRows[title].length > 6 ){
                throw new Error('error state2');
            } 
        }
    });
})

fs.writeFileSync('out.json',JSON.stringify({
    rows: newRows,
    modified
}));
const outputLines = Object.keys(newRows).map((key) => {
    const url = newRows[key][0];
    newRows[key][0] = `[${url} url]`;
    return newRows[key].join(',')
});
fs.writeFileSync('out.csv', `${header.replace(/\r/,'').trim()},"Possible contact","Repeat entry","Types of error"\n${outputLines.join('\n')}`);
console.log(
    `Dear ${Object.keys(newRows).map((key) => newRows[key][3]).join(', ')}, @[[User:Jon (WMF)]]

    Apologies for writing to you in English. I am currently trying to fix recurring errors in scripts across our movement concerning scripts on your projects.
   
Please see the most recent report of errors in gadgets and user scripts above and help me fix them. Please don't hesitate to ask questions if you have any.

 ~~~~`);