Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Git Log Parser & Aggregator
/* eslint-env node */
/* eslint-disable no-console, dot-notation */
const fse = require('fs-extra');
const readline = require('readline');
// const platform = 'mw_less';
// const platform = 'mw_scss';
const platform = 'cosmos';
console.log(`Processing git-log file for ${platform} started...`);
const commits = [];
let commit = false;
const lineReader = readline.createInterface({
input: fse.createReadStream(`data-raw/git-log-stat-${platform}.txt`)
});
lineReader.on('line', function (line) {
// ignore empty lines
if (line === '' || line === '\n') {
return;
}
// we've found a new commit!
if (line.match(/^commit/)) {
// push the previous commit in the store (if is not the initial "false" commit)
if(commit) {
commits.push(commit);
}
// initialise a new commit
commit = {
merge: false,
exclude: false,
};
// get the hash value
commit.hash = line.match(/^commit (.*)/)[1];
// console.log('Found hash:', commit.hash);
}
if (line.match(/^Author:/)) {
// get the author name
commit.author = line.match(/^Author: (.*) <(.*)>/)[1];
// console.log('Found author:', commit.author);
}
if (line.match(/^Date:/)) {
// get the commit date
commit.date = line.match(/^Date:\s+(.*)/)[1];
// console.log('Found date:', commit.date);
}
if (line.match(/^Merge:/)) {
// is a merge
commit.merge = true;
// console.log('Found a merge');
}
if (line.match(/^ [^\#]/)) {
// get the (first line of) commit message
if(!commit.message) {
commit.message = line.match(/^ (.*)/)[1];
}
// console.log('Found a message', commit.message);
}
if (line.match(/\d+ file[s]? changed/)) { // in case of merge this line doesn't exist
// get the file changes => X files changed, Y insertions(+), Z deletions(-)
line.split(', ').forEach(function(change) {
if(change.match(/\d+ file[s]? changed/)) {
commit.files = change.match(/(\d+) file[s]? changed/)[1];
// console.log('Files changed', commit.files);
}
if(change.match(/\d+ insertion[s]?\(\+\)/)) {
commit.insertions = change.match(/(\d+) insertion[s]?\(\+\)/)[1];
// console.log('Insertions', commit.insertions);
}
if(change.match(/\d+ deletion[s]?\(-\)/)) {
commit.deletions = change.match(/(\d+) deletion[s]?\(-\)/)[1];
// console.log('deletions', commit.deletions);
}
});
}
});
lineReader.on('close', () => {
// write commits data as tab-separated values
let output = '';
commits.forEach(function(commit) {
const hash = commit.hash;
const author = commit.author;
const date = commit.date;
const merge = commit.merge ? 1 : 0;
const message = commit.message;
const files = commit.files ? parseInt(commit.files, 10) : 0;
const insertions = commit.insertions ? parseInt(commit.insertions, 10) : 0;
const deletions = commit.deletions ? parseInt(commit.deletions, 10) : 0;
const row = `${hash}\t${author}\t${date}\t${merge}\t${message}\t${files}\t${insertions}\t${deletions}`;
output += `${row}\n`;
// console.log(row);
});
fse.writeFileSync(`./data-processed/commits-${platform}.tsv`, output);
// write commits data as JSON file
fse.writeJsonSync(`./data-processed/commits-${platform}.json`, commits);
// write aggregated commits data as JSON file
let aggregated = {};
commits.forEach(function(commit) {
if (commit.merge) {
return;
}
// blacklisting
if(platform === 'mw_less') {
if(
// initial project setup
commit.message.match(/^\[rework\]/) ||
...
// remove less codebase
commit.message.match(/^\[MW-****\]/)
) {
return;
}
}
if(platform === 'mw_scss') {
if(
// less to sass conversion
commit.message.match(/^\[MW-****\]/) ||
// code styling
commit.message.match(/^\[MW-****\]/) ||
commit.message.match(/^\[MW-****\]/) ||
commit.message.match(/^\[MW-****\]/) ||
...
) {
return;
}
}
if(platform === 'cosmos') {
if(
...
) {
return;
}
}
const date = commit.date;
let newFiles;
let newInsertions;
let newDeletions;
const commitFiles = commit.files ? parseInt(commit.files, 10) : 0;
const commitInsertions = commit.insertions ? parseInt(commit.insertions, 10) : 0;
const commitDeletions = commit.deletions ? parseInt(commit.deletions, 10) : 0;
if(aggregated[date]) {
newFiles = aggregated[date]['files'] + commitFiles;
newInsertions = aggregated[date]['insertions'] + commitInsertions;
newDeletions = aggregated[date]['deletions'] + commitDeletions;
} else {
newFiles = commitFiles;
newInsertions = commitInsertions;
newDeletions = commitDeletions;
}
aggregated[date] = {
date: date,
files: newFiles,
insertions: newInsertions,
deletions: newDeletions,
};
});
const aggregatedArr = [];
let aggregatedTxt = 'date\tfiles\tinsertions\tdeletions\n';
Object.keys(aggregated).forEach((date) => {
const row = `${date}\t${aggregated[date].files}\t${aggregated[date].insertions}\t${aggregated[date].deletions}`;
aggregatedTxt += `${row}\n`;
aggregatedArr.push(aggregated[date]);
});
fse.writeFileSync(`./data-processed/aggregated-${platform}.tsv`, aggregatedTxt);
fse.writeJsonSync(`./data-processed/aggregated-${platform}.json`, aggregatedArr);
fse.writeJsonSync(`./application/src/data/aggregated-${platform}.json`, aggregatedArr);
console.log(`Processing git-log file for ${platform} completed.`);
});
@didoo

This comment has been minimized.

Copy link
Owner Author

didoo commented Jan 26, 2019

A script to parse and process the git-log of a repository and output meaningful data to be visualized in a D3.js graph.

More details here: https://medium.com/@didoo/measuring-the-impact-of-a-design-system-7f925af090f7

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.