Generate an HTML page with graphs representing the evolution of issues & PRs for a given GitHub repo

build_stats_page.py

#!/usr/bin/env python3

# Inspired by https://nf-co.re/stats#github_prs
# Original code by Phil Ewens, cf. https://github.com/nf-core/nf-co.re/issues/190
# This code is under MIT License, as the original nf-co.re website is

# INSTALL: pip install jinja2 livereload requests xreload
# USAGE: ./build_stats_page.py [--use-dump] [--watch-and-serve] org/repo

import argparse, json, os, sys, types, webbrowser
from collections import defaultdict
from datetime import datetime
from itertools import count
from os.path import basename, dirname, join, realpath

from jinja2 import Environment, FileSystemLoader
from livereload import Server
import requests
from xreload import xreload


PARENT_DIR = dirname(realpath(__file__))


def main():
    args = parse_args()
    build_page(args)
    if args.watch_and_serve:
        watch_and_serve(args)

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('org_repo')
    parser.add_argument('--use-dump', action='store_true')
    parser.add_argument('--watch-and-serve', action='store_true')
    return parser.parse_args()  # reads sys.argv

def build_page(args):
    github_stats = read_dump()
    if not github_stats or not args.use_dump:
        github_stats = get_github_stats(args.org_repo)
        write_dump(github_stats)
    generate_html(pre_process(github_stats))

def get_github_stats(org_repo):
    github_stats = {'org_repo': org_repo, 'issues': []}
    headers = {'Authorization': 'token ' + os.environ['GITHUB_OAUTH_TOKEN']} if 'GITHUB_OAUTH_TOKEN' in os.environ else None
    for page in count(1):
        response = requests.get('https://api.github.com/repos/{}/issues?state=all&page={}'.format(org_repo, page), headers=headers)
        if response.status_code == 403:
            print(response.json(), file=sys.stderr)
        response.raise_for_status()
        issues = response.json()
        if issues:
            github_stats['issues'].extend(issues)
        else:
            break
    return github_stats

def pre_process(github_stats):
    issues_per_creation_day = defaultdict(list)
    for issue in github_stats['issues']:
        creation_day = issue['created_at'].split('T')[0]
        issues_per_creation_day[creation_day].append(issue)
    open_issues_count_over_time, closed_issues_count_over_time = [], []
    open_prs_count_over_time, closed_prs_count_over_time = [], []
    open_issues_count, closed_issues_count = 0, 0
    open_prs_count, closed_prs_count = 0, 0
    for day in sorted(issues_per_creation_day.keys()):
        open_issues_count += len([issue for issue in issues_per_creation_day[day]
                                        if 'pull_request' not in issue])
        open_issues_count_over_time.append((day, open_issues_count))
        closed_issues_count += len([issue for issue in issues_per_creation_day[day]
                                          if 'pull_request' not in issue and issue['state'] == 'closed'])
        closed_issues_count_over_time.append((day, closed_issues_count))
        open_prs_count += len([issue for issue in issues_per_creation_day[day]
                                     if 'pull_request' in issue])
        open_prs_count_over_time.append((day, open_prs_count))
        closed_prs_count += len([issue for issue in issues_per_creation_day[day]
                                       if 'pull_request' in issue and issue['state'] == 'closed'])
        closed_prs_count_over_time.append((day, closed_prs_count))
    github_stats['open_issues_count_over_time'] = open_issues_count_over_time
    github_stats['closed_issues_count_over_time'] = closed_issues_count_over_time
    github_stats['open_prs_count_over_time'] = open_prs_count_over_time
    github_stats['closed_prs_count_over_time'] = closed_prs_count_over_time
    return github_stats

def write_dump(dump_data):
    with open(join(PARENT_DIR, 'dump.json'), 'w+') as json_file:
        return json.dump(dump_data, json_file)

def read_dump():
    try:
        with open(join(PARENT_DIR, 'dump.json')) as json_file:
            return json.load(json_file)
    except (FileNotFoundError, json.JSONDecodeError):
        return None

def generate_html(github_stats):
    env = Environment(loader=FileSystemLoader(PARENT_DIR))
    template = env.get_template('stats_template.html')
    template.globals['now'] = datetime.utcnow
    with open(join(PARENT_DIR, 'stats.html'), 'w') as output_file:
        output_file.write(template.render(github_stats))

def watch_and_serve(args):
    server = Server()
    server.watch('stats_template.html', lambda: build_page(args))
    server.watch(__file__,              lambda: reload_script() and build_page(args))
    webbrowser.open('http://localhost:5500/stats.html')
    server.serve(root=PARENT_DIR, port=5500)

def reload_script():
    return xreload(sys.modules[__name__], new_annotations={'RELOADING': True})


if __name__ == '__main__' and 'RELOADING' not in __annotations__:
    main()

stats_template.html

<!doctype html>
<html lang="en">
  <head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
    <title>GitHub repository issues &amp; pull requests statistics</title>
    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
    <link href="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.8.0/Chart.min.css" rel="stylesheet">
    <script src="https://kit.fontawesome.com/471b59d3f8.js"></script>
    <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.24.0/moment.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.8.0/Chart.min.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-zoom@0.7.3/dist/chartjs-plugin-zoom.min.js"></script>
    <style>
@import url('https://fonts.googleapis.com/css?family=Maven+Pro:400,700|Open+Sans:400,700');
body {
  color: #444444;
  font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
}
h1, h2, h3, h4, h5 {
  font-family: 'Maven Pro', sans-serif;
  color: #159957;
  font-weight: 700;
}
a {
  color: #1e6bb8;
}

.triangle {
  position: relative;
}
.triangle:before {
  background-repeat: no-repeat;
  background-size: 100% 100%;
  content: '';
  display: block;
  width: 100%;
  left: 0;
  height: 30px;
}
.triangle-down:before {
  background-image: url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='100' height='100' viewBox='0 0 100 100' preserveAspectRatio='none'%3E%3Cpolygon points='100,0 50,100 0,0' style='fill:%2322ae63;' /%3E%3C/svg%3E");
}
.subheader-triangle-down:before {
  background-image: url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='100' height='100' viewBox='0 0 100 100' preserveAspectRatio='none'%3E%3Cpolygon points='100,0 50,100 0,0' style='fill:%23ededed;' /%3E%3C/svg%3E");
}

.mainpage-heading {
  background-color: #22ae63;
  color: #ffffff;
  padding: 3rem 0;
}
.mainpage-heading h1, .mainpage-heading h1 a {
  color: #ffffff;
  text-decoration: none;
}
@media only screen and (max-width: 770px) {
  .mainpage-heading h1 {
    font-size: 3rem;
  }
}
.mainpage-heading h1 a:hover, .mainpage-heading h1 a:focus {
  color: #dae0e5;
}
.mainpage-heading code {
  color: #ffffff;
}
.main-content {
  min-height: 300px;
  margin-top: 1rem;
  margin-bottom: 6rem;
  font-size: 1.1rem;
  line-height: 1.6;
}
.main-content h1, .main-content h2, .main-content h3,
.main-content h4, .main-content h5, .main-content h6 {
  font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
  font-weight: 500;
  padding-top: 60px;
  margin-top: -50px;
}
.main-content h1 {
  border-bottom: 1px solid #22ae63;
  margin-top: -30px;
}
footer {
  margin-bottom: 4rem;
}
    </style>
  </head>
  <body>

    <div class="mainpage">
      <div class="mainpage-heading">
        <div class="container">
          <h1 class="display-3"><a href="https://github.com/{{org_repo}}">{{org_repo}}</a> in numbers</h1>
          <p class="lead">Generated on {{now().strftime('%c')}}</p>
        </div>
      </div>

      <div class="triangle triangle-down"></div>
      <div class="container main-content">

        <div class="row">
          <div class="col-lg-6">

            <h2 class="mt-0" id="github_prs">Pull Requests</h2>
            <div class="card bg-light mt-4">
              <div class="card-body">
                <canvas id="github_prs_plot" height="200"></canvas>
                <p class="card-text small text-muted">
                  <a href="#" data-target="github_prs" class="reset_chart_zoom text-muted"><i class="fas fa-search-minus"></i> Reset zoom</a>
                </p>
              </div>
            </div>

          </div>
          <div class="col-lg-6">

            <h2 class="mt-0" id="github_issues">Issues</h2>
            <div class="card bg-light mt-4">
              <div class="card-body">
                <canvas id="github_issues_plot" height="200"></canvas>
                <p class="card-text small text-muted">
                  <a href="#" data-target="github_issues" class="reset_chart_zoom text-muted"><i class="fas fa-search-minus"></i> Reset zoom</a>
                </p>
              </div>
            </div>

          </div>
        </div>

      </div>
    </div>
    
    <footer class="container text-center">
        Generated by <a href="https://gist.github.com/Lucas-C/5c9730756e7b3f795c6d121d38a9ce88">build_stats_page.py</a>
    </footer>

<script type="text/javascript">
$(function (){
  // Placeholder for chart data
  var chartData = {};
  var charts = {};

  // Chart.JS base config
  var baseChartConfig = {
    type: 'line',
    options: {
      title: {
        display: true,
        fontSize: 16
      },
      elements: {
        line: {
          borderWidth: 1,
          fill: '-1', // by default, fill lines to the previous dataset
          tension: 0 // disables bezier curves
        }
      },
      scales: {
        xAxes: [{
          type: 'time',
          time: { minUnit: 'day' }
        }],
      },
      legend: {
        position: 'bottom',
        labels: { lineWidth: 1 }
      },
      tooltips: { mode: 'x' },
      plugins: {
        zoom: {
          zoom: {
            enabled: true,
            drag: true,
            mode: 'x',
            speed: 0.05
          }
        }
      }
    }
  };

  // GitHub Pull Requests chart
  chartData['github_prs'] = JSON.parse(JSON.stringify(baseChartConfig));
  chartData['github_prs'].data = {
    datasets: [
      {
        label: 'Closed / Merged',
        backgroundColor: 'rgba(104, 72, 186, 0.2)',
        borderColor: 'rgba(104, 72, 186, 1)',
        pointRadius: 0,
        fill: 'origin',  // explicitly fill the first dataset to the x axis
        data: [
        {% for x, y in closed_prs_count_over_time %}
          { x: "{{x}}", y: {{y}} },{% endfor %}
        ]
      },
      {
        label: 'Open',
        backgroundColor: 'rgba(83, 164, 81, 0.2)',
        borderColor: 'rgba(83, 164, 81, 1)',
        pointRadius: 0,
        data: [
        {% for x, y in open_prs_count_over_time %}
          { x: "{{x}}", y: {{y}} },{% endfor %}
        ]
      }
    ]
  };
  chartData['github_prs'].options.title.text = 'GitHub Pull Requests over time';
  chartData['github_prs'].options.elements.line.fill = '-1'; // by default, fill lines to the previous dataset
  chartData['github_prs'].options.legend = {
    position: 'bottom',
    labels: { lineWidth: 1 }
  };
  var ctx = document.getElementById('github_prs_plot').getContext('2d');
  charts['github_prs'] = new Chart(ctx, chartData['github_prs']);


  // GitHub issues chart
  chartData['github_issues'] = JSON.parse(JSON.stringify(baseChartConfig));
  chartData['github_issues'].data = {
    datasets: [
      {
        label: 'Closed',
        backgroundColor: 'rgba(199, 70, 78, 0.2)',
        borderColor: 'rgba(199, 70, 78, 1)',
        pointRadius: 0,
        fill: 'origin',  // explicitly fill the first dataset to the x axis
        data: [
        {% for x, y in closed_issues_count_over_time %}
          { x: "{{x}}", y: {{y}} },{% endfor %}
        ]
      },
      {
        label: 'Open',
        backgroundColor: 'rgba(83, 164, 81, 0.2)',
        borderColor: 'rgba(83, 164, 81, 1)',
        pointRadius: 0,
        data: [
        {% for x, y in open_issues_count_over_time %}
          { x: "{{x}}", y: {{y}} },{% endfor %}
        ]
      }
    ]
  };
  chartData['github_issues'].options.title.text = 'GitHub Issues over time';
  chartData['github_issues'].options.elements.line.fill = '-1'; // by default, fill lines to the previous dataset
  chartData['github_issues'].options.legend = {
    position: 'bottom',
    labels: { lineWidth: 1 }
  };
  var ctx = document.getElementById('github_issues_plot').getContext('2d');
  charts['github_issues'] = new Chart(ctx, chartData['github_issues']);

  $('.reset_chart_zoom').click(function(e){
    e.preventDefault();
    var target = $(this).data('target');
    charts[target].resetZoom();
  });
});
</script>

  </body>
</html>