loc.py

from shell import shell
from datetime import date, timedelta
import json

from concurrent.futures import ThreadPoolExecutor


import csv

DAYS_IN_PAST = 180
PROGRESS_EVERY = 5


def asli_shell(cmd):
    return shell(f'sh -c "{cmd}"')


def dates_to_calculate():
    def _iter():
        for days_ago in range(1, DAYS_IN_PAST):
            yield date.today() - timedelta(days=days_ago)

    return list(_iter())


# {
#   "alerts": [
#     {"date": "29/01/1993", "count": 234},
#     {"date": "30/01/1993", "count": 235}
#   ],
#   "rms": [
#     {"date": "29/01/1993", "count": 234},
#     {"date": "30/01/1993", "count": 235}
#   ]
# }
def get_loc_map(services, dates):
    loc_map = {}

    def do(service):
        loc_map[service] = []
        for i, current_date in enumerate(dates):
            if i % PROGRESS_EVERY == 0:
                print(f"[{i}/{len(dates)}] {service}")

            r = shell(
                f"git -C {service} rev-list -n 1 --first-parent --before='{current_date}' master"
            )
            assert r.code == 0
            commit = r.output(raw=True).strip()

            r = shell(f"git -C {service} checkout {commit}")
            assert r.code == 0

            cloc_json = asli_shell(f"cloc --include-ext=py --json {service}").output(
                raw=True
            )
            line_count = json.loads(cloc_json)["SUM"]["code"]
            loc_map[service].append({"date": date, "count": line_count})

    with ThreadPoolExecutor(max_workers=8) as executor:
        futures = []

        for service in services:
            futures.append(executor.submit(do, service))

        for future in futures:
            future.result()

    return loc_map


services = asli_shell("git submodule status | awk '{print $2}'").output()
dates = dates_to_calculate()
loc_map = get_loc_map(services, dates)

with open("loc.csv", "w", newline="") as csvfile:
    writer = csv.writer(csvfile, delimiter=",")
    writer.writerow(["SERVICE"] + [str(date) for date in dates])
    for service, count_data_arr in loc_map.items():
        writer.writerow(
            [service] + list(reversed([data["count"] for data in count_data_arr]))
        )