Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Backporting of PRs and language changes
"""
Put this file in a master checkout under .github/.
It should be next to backport.py.
"""
import argparse
import glob
import subprocess
import shlex
def backport_language(language_file, blacklisted_ids, diff_to_stdout=False):
result = subprocess.run(
shlex.split(
"git diff HEAD..upstream/master -- %s" % language_file
), check=True, stdout=subprocess.PIPE)
input_lines = []
chunk = []
# We start with this set to True, to pick up any headers before the
# patch really begins
chunk_has_modification = True
# Decode the result, skip the 4 line header
for line in result.stdout.decode().split('\n'):
if not line or line.startswith('@@') or line.startswith(('---', '+++')):
# Only add the chunk if there was a modification to it.
# 'git apply' cannot handle chunks with no modifications.
if chunk_has_modification:
input_lines.extend(chunk)
chunk = []
chunk_has_modification = False
# Check for the start of a new chunk
if line.startswith('@@'):
chunk.append(line)
else:
input_lines.append(line)
continue
# Passthrough all the unmodified lines (they are just context)
if not line.startswith(('-', '+')):
chunk.append(line)
continue
id = line[1:].split(':')[0]
if id not in blacklisted_ids:
# Modification is not blacklisted; this is fine
chunk_has_modification = True
chunk.append(line)
continue
# A blacklisted id; skip the modification
if line.startswith('+'):
pass
else:
chunk.append(' ' + line[1:])
# No chunks found, so nothing to do
if len(input_lines) < 6:
return
total_input = "\n".join(input_lines)
if diff_to_stdout:
print(total_input)
return
result = subprocess.run(
shlex.split(
"git apply --recount"
), check=True, input=total_input.encode())
def create_blacklisted_ids():
# First check what changed in english.txt. Every change is blacklisted and
# translations in these lines will not be backported
result = subprocess.run(
shlex.split(
"git diff HEAD..upstream/master -- src/lang/english.txt"
), check=True, stdout=subprocess.PIPE)
blacklisted_ids = []
# Walk the diff line by line
for line in result.stdout.decode().split('\n'):
# Ignore headers
if line.startswith(('---', '+++')) or not line:
continue
# Find all the lines that are modified
if line.startswith(('-', '+')):
# Store that id in a blacklist
id = line[1:].split(':')[0]
blacklisted_ids.append(id)
return blacklisted_ids
def parse_command_line():
parser = argparse.ArgumentParser(description='Backport languages from master to release branch')
parser.add_argument('languages', metavar='LANGUAGE', type=str, nargs='*', help='which languages to backport (empty for all)')
parser.add_argument('--diff', action='store_true', help='only show the diff; do not apply')
return parser.parse_args()
def main():
args = parse_command_line()
blacklisted_ids = create_blacklisted_ids()
if args.languages:
language_files = [ "src/lang/%s.txt" % language for language in args.languages ]
else:
language_files = glob.glob("src/lang/*.txt") + glob.glob("src/lang/unfinished/*.txt")
for language_file in language_files:
print("Backporting %s ..." % language_file[len("src/lang/"):])
backport_language(language_file, blacklisted_ids, diff_to_stdout=args.diff)
if __name__ == "__main__":
main()
"""
Put this file in a master checkout under .github/.
It should be next to backport-languages.py.
This assumes your git "origin" points to your fork, and "upstream" to upstream.
This will force-push to a branch called "release-backport".
Execute with:
$ python3 .github/backport.py
And follow the instructions. After the PR is merged, run:
$ python3 .github/backport.py --mark-done <PR-NUMBER>
"""
import json
import os
import subprocess
import sys
# NOTE: Replace this with your own toekn
BEARER_TOKEN = "ghp_???"
# NOTE: Replace this with your own GitHub username
USERNAME = "TrueBrain"
pr_query = """
query ($number: Int!) {
repository(owner: "OpenTTD", name: "OpenTTD") {
pullRequest(number: $number) {
body
}
}
}
"""
pr_search_query = """
query ($search: String!) {
search(query: $search, type: ISSUE, first: 100) {
issueCount
edges {
node {
... on PullRequest {
number
title
commits(first: 100) {
totalCount
}
mergedAt
mergeCommit {
oid
}
}
}
}
}
}
"""
def do_query(query, variables):
query = query.replace("\n", "").replace('\\', '\\\\').replace('"', '\\"')
variables = json.dumps(variables).replace('\\', '\\\\').replace('"', '\\"')
res = subprocess.run(["curl", "-H", "Authorization: bearer " + BEARER_TOKEN, "-X", "POST", "-d", '{"query": "' + query + '", "variables": "' + variables + '"}', "https://api.github.com/graphql"], capture_output=True)
if res.returncode != 0:
return None
return json.loads(res.stdout)
def do_remove_label(number):
return subprocess.run(["curl", "-H", "Authorization: bearer " + BEARER_TOKEN, "-X", "DELETE", f"https://api.github.com/repos/OpenTTD/OpenTTD/issues/{number}/labels/backport%20requested"], capture_output=True)
def do_add_label(number):
return subprocess.run(["curl", "-H", "Authorization: bearer " + BEARER_TOKEN, "-X", "POST", "-d", '{"labels": ["backported"]}', f"https://api.github.com/repos/OpenTTD/OpenTTD/issues/{number}/labels"], capture_output=True)
def do_command(command):
return subprocess.run(command, capture_output=True)
def main():
if len(sys.argv) > 1 and sys.argv[1] == "--mark-done":
backport_pr = do_query(pr_query, {"number": int(sys.argv[2])})
if backport_pr is None:
print("ERROR: couldn't fetch backport PR")
return
for line in backport_pr["data"]["repository"]["pullRequest"]["body"].split("\n"):
if line.startswith("<!-- Backported: "):
prs = [int(pr) for pr in line.split(":")[1].split(" ")[1].split(",")]
print("Update labels from backported PRs")
for pr in prs:
print(f"- #{pr} ..")
res = do_remove_label(pr)
if res.returncode != 0:
print(f"ERROR: failed to remove label from {pr}")
res = do_add_label(pr)
if res.returncode != 0:
print(f"ERROR: failed to add label to {pr}")
print("All done")
return
dont_push = False
if len(sys.argv) > 1 and sys.argv[1] == "--dont-push":
dont_push = True
resume = None
resume_i = None
if os.path.exists(".backport-resume"):
with open(".backport-resume", "r") as fp:
resume_str, _, resume_i_str = fp.read().partition(",")
resume = int(resume_str)
resume_i = int(resume_i_str)
print(f"Resuming backporting from {resume}")
all_prs = do_query(pr_search_query, {"search": "is:closed is:pr label:\"backport requested\" repo:OpenTTD/OpenTTD"})
if all_prs is None:
print("ERROR: couldn't fetch all Pull Requests marked for 'backport requested'")
return
if not resume:
do_command(["git", "fetch", "upstream"])
do_command(["git", "checkout", "upstream/release/12", "-B", "release-backport"])
for pr in sorted(all_prs["data"]["search"]["edges"], key=lambda x: x["node"]["mergedAt"]):
if resume:
if resume != pr['node']['number']:
continue
resume = None
print(f"Merging #{pr['node']['number']}: {pr['node']['title']} (resuming)")
else:
print(f"Merging #{pr['node']['number']}: {pr['node']['title']}")
for i in range(pr["node"]["commits"]["totalCount"]):
if resume_i is not None:
if resume_i != i:
continue
resume_i = None
continue
commit = pr["node"]["commits"]["totalCount"] - i - 1
commit_str = f'{pr["node"]["mergeCommit"]["oid"]}' + "".join(["^"] * commit)
print(f' Commit #{i}: {commit_str} ...')
res = do_command(["git", "cherry-pick", commit_str])
if res.returncode != 0:
with open(".backport-resume", "w") as fp:
fp.write(str(pr['node']['number']) + "," + str(i))
print(res.stdout.decode())
print("")
print("Cherry-pick failed: please fix the issue manually and run script again.")
return
if os.path.exists(".backport-resume"):
os.unlink(".backport-resume")
print("")
print("Done cherry-picking")
print("Backporting language changes")
res = do_command(["python3", ".github/backport-languages.py"])
if res.returncode != 0:
print("ERROR: backporting language changes failed")
return
do_command(["git", "add", "src/lang/*.txt"])
do_command(["git", "commit", "-m", "Update: Backport language changes"])
print("Done backporting language changes")
print("")
print("Your commit message:")
print("")
marker = []
print("## Description")
print("Backport of all closed Pull Requests labeled as 'backport requested' into `release/12`.")
for pr in sorted(all_prs["data"]["search"]["edges"], key=lambda x: x["node"]["mergedAt"]):
print(f"- https://github.com/OpenTTD/OpenTTD/pull/{pr['node']['number']}")
marker.append(str(pr['node']['number']))
print("- All language changes")
print(f"<!-- Backported: {','.join(marker)} -->")
print("")
if dont_push:
print("Done with backport; you can now push this branch to remote:")
print(" git push -f --set-upstream origin release-backport")
print("After that, go to this URL:")
else:
res = do_command(["git", "push", "-f", "--set-upstream", "origin", "release-backport"])
if res.returncode != 0:
print("ERROR: failed to push to remote")
else:
print("Pushed to remote")
print("You can create the PR here:")
print(" https://github.com/OpenTTD/OpenTTD/compare/release/12..." + USERNAME + ":release-backport?expand=1&title=Backport%20master%20into%20release%2f12")
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment