Skip to content

Instantly share code, notes, and snippets.

Last active May 25, 2022 11:40
Show Gist options
  • Star 14 You must be signed in to star a gist
  • Fork 4 You must be signed in to fork a gist
  • Save plajjan/42592665afd5ae045ee36220e19919aa to your computer and use it in GitHub Desktop.
Save plajjan/42592665afd5ae045ee36220e19919aa to your computer and use it in GitHub Desktop.
Git automation stuff
- build
- mr-robot
NCS_VERSION: "4.2.3"
FILE: "nso-${NCS_VERSION}.linux.x86_64.installer.bin"
.build.def: &build_def
stage: build
- docker
- if [[ ${CI_COMMIT_REF_NAME} != 'master' ]]; then export ADDITIONAL_TAG=-${CI_COMMIT_REF_NAME}; fi
- git lfs fetch -I ${FILE}
- git lfs checkout ${FILE}
- make build-image
- make tag-image
- docker login -u gitlab-ci-token -p ${CI_BUILD_TOKEN} ${CI_REGISTRY}
- make push-image
# only push master to external repo
- if [[ ${CI_COMMIT_REF_NAME} == 'master' ]]; then make push-image; fi
# fugly hack for git lfs problems in gitlab ci
- rm .git/hooks/post-checkout || true
.mr-robot-def: &mr-robot-def
stage: mr-robot
- docker
- master@TeraStream/cisco-nso
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
- eval $(ssh-agent -s)
- ssh-add <(echo "$GIT_SSH_PRIV_KEY")
- git config --global ""
- git config --global "Mr. Robot"
- mkdir -p ~/.ssh
- cat gitlab-known-hosts >> ~/.ssh/known_hosts
- echo -e "\n\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\nHello, I'm Mr. Robot and I will assist you with boring git tasks.\n\nI will now try to clone TeraStream/nso-ts, modify the NSO version used and submit a MR.\n\n//Mr. Robot\n\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n"
# extract NCS tar
- bash ${FILE} --local-install nso-${NCS_VERSION}
# get list of tail-f issues
- pip3 install rt
- utils/ --list-issues > tailf-issues
# prepare MR description based on tailf-issues and the changelog
- bash utils/ tailf-issues nso-${NCS_VERSION}/CHANGES > mr-description
# clone nso-ts
- git clone
- cd nso-ts
- git fetch origin
- git branch -r
- git checkout -b nso-${NCS_VERSION} origin/nso-${NCS_VERSION} || git checkout -b nso-${NCS_VERSION}
# update NCS version in relevant files
- sed -i Dockerfile -e "s/^\(FROM.*\):[0-9]\+\(\.[0-9]\+\)\+$/\1:${NCS_VERSION}/"
- sed -i Makefile -e "s/^NCS_VERSION=.*/NCS_VERSION=${NCS_VERSION}/"
- sed -i .gitlab-ci.yml -e "s/^\( NCS_VERSION:[ ]\).*/\1\"${NCS_VERSION}\"/"
# if we are the NCS version in use there will be no diff
- git diff
# only commit if there are changes on this branch
# if we in the checkout above checked out a remote branch, then it is
# probably identical to this one so there will be nothing to commit
- git diff --exit-code || git commit -a -m "Use NSO v${NCS_VERSION}"
- git rebase master
# only push if we are different from master
- git diff --exit-code master || git push --force origin nso-${NCS_VERSION}
- cd ..
- sleep 5 # let CI start up
- 'utils/ --project TeraStream/nso-ts --source-branch nso-${NCS_VERSION} --title "WIP: Use NSO v${NCS_VERSION}" --description-file mr-description'
<<: *build_def
NCS_VERSION: "4.2.3"
<<: *build_def
NCS_VERSION: "4.2.4"
<<: *build_def
NCS_VERSION: "4.3.3"
<<: *build_def
NCS_VERSION: "4.4.3"
<<: *mr-robot-def
NCS_VERSION: "4.2.3"
<<: *mr-robot-def
NCS_VERSION: "4.2.4"
<<: *mr-robot-def
NCS_VERSION: "4.3.3"
<<: *mr-robot-def
NCS_VERSION: "4.4.3"
#!/usr/bin/env python3
import json
import os
import re
import requests
import urllib.parse
class OpenMr:
def __init__(self, project_name):
self.base_url = ""
self.api_url = ""
self.token = os.getenv("PRIVATE_TOKEN")
self.headers = { 'PRIVATE-TOKEN': self.token }
self.project_name = project_name
self.project_id = self.get_project_id(self.project_name)
self.project_url = "/projects/{}".format(self.project_id)
def path_guess(self, path):
""" Guess what the URL should be based on an input path
if re.match('[A-Za-z]+://', path):
url = path
url = "{}{}".format(self.api_url, path)
return url
def http_get(self, path):
""" Simple HTTP get on path relative to base_url and doing
authentication using our token
url = self.path_guess(path)
return requests.get(url, headers=self.headers)
def http_post(self, path, data):
""" Simple HTTP get on path relative to base_url and doing
authentication using our token
url = self.path_guess(path)
return, headers=self.headers, data=data)
def get_json(self, path):
""" Just do a HTTP get, assume it goes fine, and return the data as a
JSON struct
r = self.http_get(path)
data = json.loads(r.text)
return data
def urlencode(self, text):
""" URL encode a string
return urllib.parse.urlencode({'a': text})[2:]
def get_project_id(self, name):
""" Get GitLab project ID based on project path, like 'TeraStream/nso-ts'
return int(self.get_json("/projects/{}".format(self.urlencode(name)))['id'])
def get_mr(self, mrid):
data = self.get_json(self.project_url + "/merge_requests/{}".format(mrid))
def create_mr(self, source_branch, target_branch, title, description):
data = {
'id': self.project_id,
'source_branch': source_branch,
'target_branch': target_branch,
'remove_source_branch': True,
'title': title,
'description': description
self.http_post(self.project_url + "/merge_requests", data=data)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--project', required=True, help="Project path to retry jobs for, e.g. 'TeraStream/nso-ts'")
parser.add_argument('--title', required=True, help="Title of MR")
parser.add_argument('--description-file', required=True, help="File containing MR description")
parser.add_argument('--source-branch', required=True, help="Source branch")
# TODO: could find out what the default branch is for the repo
parser.add_argument('--target-branch', default='master', help="Target branch")
args = parser.parse_args()
f = open(args.description_file, encoding='utf-8')
desc =
mr = OpenMr(args.project)
if mr.get_mr(args.source_branch) is None:
mr.create_mr(args.source_branch, args.target_branch, args.title, desc)
#!/usr/bin/env bash
# This is our first script to open an MR. I've since written a - try to use it instead!
# Extract the host where the server is running, and add the URL to the APIs
[[ $HOST =~ ^https?://[^/]+ ]] && HOST="${BASH_REMATCH[0]}/api/v4/projects/"
CI_PROJECT_NAME_URLENCODED=$(echo ${CI_PROJECT_NAME} | python3 -c "import sys, urllib.parse; print(urllib.parse.urlencode({'a':})[2:])")
CI_PROJECT_ID=$(curl --silent --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}" ${HOST}${CI_PROJECT_NAME_URLENCODED} | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
# Look which is the default branch
TARGET_BRANCH=$(curl --silent "${HOST}${CI_PROJECT_ID}" --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}" | python3 -c "import sys, json; print(json.load(sys.stdin)['default_branch'])")
# The description of our new MR, we want to remove the branch after the MR has
# been closed
\"id\": ${CI_PROJECT_ID},
\"source_branch\": \"${CI_COMMIT_REF_NAME}\",
\"target_branch\": \"${TARGET_BRANCH}\",
\"remove_source_branch\": true,
\"title\": \"Robot: ${CI_COMMIT_REF_NAME}\",
# Require a list of all the merge request and take a look if there is already
# one with the same source branch
LISTMR=`curl --silent "${HOST}${CI_PROJECT_ID}/merge_requests?state=opened" --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}"`;
COUNTBRANCHES=`echo ${LISTMR} | grep -o "\"source_branch\":\"${CI_COMMIT_REF_NAME}\"" | wc -l`;
# No MR found, let's create a new one
if [ ${COUNTBRANCHES} -eq "0" ]; then
curl -X POST "${HOST}${CI_PROJECT_ID}/merge_requests" \
--header "Content-Type: application/json" \
--data "${BODY}";
echo "Opened a new merge request: ${CI_COMMIT_REF_NAME} and assigned to you";
echo "No new merge request opened";
# this takes a list of Tail-F issues and converts it into a set of awk commands
# that can grep in the NCS changelog based on the issues but instead of just
# returning a single line it extracts the paragraph that mentions the issue.
for I in $(cat $1); do
echo "awk -v RS=' - ' '/${I}/ { print \" -\",\$0 }' $2";
done >
echo -e "According to the changelog, the following issues have been fixed:"
grep -f $1 $2 | sed -e 's/[^0-9]\+\([0-9][0-9][0-9]\+\)[^0-9]/ * https:\/\/\/rt\/Ticket\/Display.html?id=\1\n/g' | sed '/^$/d' | sort -n
echo -e "\nWith the following descriptions:\n\`\`\`"
echo -e "\`\`\`\n\nFull changelog:\n\`\`\`"
cat $2
echo "\`\`\`"
#!/usr/bin/env python3
import os
import rt
username = os.getenv('TAILF_RT_USER')
password = os.getenv('TAILF_RT_PASS')
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--list-issues', action='store_true')
args = parser.parse_args()
tracker = rt.Rt('', username, password)
if args.list_issues:
# XXX change the name of the queue to your queue!
for ticket in'DTAG', Status='open'):
Copy link


very good work! And it was a pleasure to read the related article at gitlab 😄

Just as a note, there seems to be a good and sufficient tool for automating the "merge request creation":

Kind regards

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment