Search for keywords at least X days old in a mercurial repo

grep-blame.py

#!/usr/bin/env python3

import argparse
import datetime
import dateutil.parser
import subprocess


def get_files_and_lines(pattern, files):
    results = {}

    # no filesnames given -------------------------
    if not files:
        pattern = ("'" + pattern + "'") if "'" not in pattern else ('"' + pattern + '"')
        command = " ".join(["grep", "-R", "-n", pattern, "*"])
        print(command)
        ret = subprocess.run(
            command,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.DEVNULL,
            encoding="UTF-8",
            errors="ignore",
            check=False,
        )
        output = ret.stdout.split("\n")

        for line in output:
            if not line:
                continue

            try:
	            sep1 = line.index(":")
	            filename = line[0:sep1]

	            sep2 = line[sep1 + 1:].index(":") + sep1 + 1
	            line_number = int(line[sep1 + 1 : sep2])

	            if filename not in results:
	                results[filename] = []
	            results[filename].append(line_number)
            except:
            	print("Error processing:", line)

        if VERBOSE:
            for filename in results.keys():
                print(filename, ":", len(results[filename]), "results")

    # explicit filenames -------------------------
    for f in files:
        ret = subprocess.run(
            ["grep", "-n", pattern, f],
            stdout=subprocess.PIPE,
            encoding="UTF-8",
            errors="ignore",
            check=False,
        )
        output = ret.stdout.split("\n")

        for line in output:
            if not line:
                continue

            sep = line.index(":")
            line_number = int(line[0:sep])

            if f not in results:
                results[f] = []
            results[f].append(line_number)

        if VERBOSE and f in results:
            print(f, ":", len(results[f]), "results")

    if VERBOSE:
        print("")

    return results


def get_lines_and_blame_data(inputs):
    results = {}

    for f in inputs.keys():
        results[f] = []

        ret = subprocess.run(
            [
                "hg",
                "blame",
                "--template",
                "{lines % '{node}<{date|shortdate}>{line}'}",
                f,
            ],
            stdout=subprocess.PIPE,
            encoding="UTF-8",
            errors="ignore",
            check=False,
        )
        if ret.returncode:
        	continue

        output = ret.stdout.split("\n")

        for line_no in inputs[f]:
            line = output[line_no - 1]
            first_sep = line.index("<")
            second_sep = line.index(">")

            changeset = line[0:first_sep]
            date = line[first_sep + 1 : second_sep]
            data = line[second_sep + 1 :]

            results[f].append((line_no, changeset, date, data))

        if VERBOSE:
            print(f, ":", len(results[f]), "lines processed")

    if VERBOSE:
        print("")

    return results


def process_results(results, days, output_file):
    if output_file:
        output_file = open(output_file, "w")

    for f in results.keys():
        for r in results[f]:
            date = dateutil.parser.parse(r[2])

            if date + datetime.timedelta(days=days) < datetime.datetime.today():
                print(f, r[0], r[2], r[3])
                if output_file:
                    output_file.write(
                        "https://searchfox.org/mozilla-central/source/%s#%s\n"
                        % (f, r[0])
                    )


VERBOSE = False
if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="grep files, and show results based on hg blame data"
    )
    parser.add_argument(
        "pattern", metavar="S", type=str, nargs=1, help="Pattern to search for"
    )
    parser.add_argument(
        "files",
        metavar="N",
        type=str,
        nargs="*",
        help="Files to process; if omitted, search all files and subdirectories.",
    )
    parser.add_argument(
        "--days",
        type=int,
        default=(30 * 18),
        help="Minimum age limit of lines to return, in days.",
    )
    parser.add_argument("--output", type=str, help="File to output searchfox links to")
    parser.add_argument("-v", action="store_true", help="verbose output")
    args = parser.parse_args()
    VERBOSE = args.v

    if VERBOSE:
        print("Searching for this pattern:", args.pattern[0])
        print("In these files:", args.files)
        print("")

    results = get_files_and_lines(args.pattern[0], args.files)
    results = get_lines_and_blame_data(results)

    process_results(results, args.days, args.output)