Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Git automation stuff
image: gitlab.dev.terastrm.net:4567/terastream/cisco-nso/ci-cisco-nso:4.2.3
stages:
- build
- mr-robot
variables:
NCS_VERSION: "4.2.3"
DOCKER_REGISTRY: "gitlab.dev.terastrm.net:4567/terastream/cisco-nso"
TARGET_REGISTRY: "repo.dev.terastrm.net:5000/"
FILE: "nso-${NCS_VERSION}.linux.x86_64.installer.bin"
.build.def: &build_def
stage: build
tags:
- docker
script:
- if [[ ${CI_COMMIT_REF_NAME} != 'master' ]]; then export ADDITIONAL_TAG=-${CI_COMMIT_REF_NAME}; fi
- git lfs fetch -I ${FILE}
- git lfs checkout ${FILE}
- make build-image
- make tag-image
- docker login -u gitlab-ci-token -p ${CI_BUILD_TOKEN} ${CI_REGISTRY}
- make push-image
# only push master to external repo
- if [[ ${CI_COMMIT_REF_NAME} == 'master' ]]; then make push-image; fi
after_script:
# fugly hack for git lfs problems in gitlab ci
- rm .git/hooks/post-checkout || true
.mr-robot-def: &mr-robot-def
stage: mr-robot
tags:
- docker
only:
- master@TeraStream/cisco-nso
before_script:
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
- eval $(ssh-agent -s)
- ssh-add <(echo "$GIT_SSH_PRIV_KEY")
- git config --global user.email "kll@dev.terastrm.net"
- git config --global user.name "Mr. Robot"
- mkdir -p ~/.ssh
- cat gitlab-known-hosts >> ~/.ssh/known_hosts
script:
- echo -e "\n\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\nHello, I'm Mr. Robot and I will assist you with boring git tasks.\n\nI will now try to clone TeraStream/nso-ts, modify the NSO version used and submit a MR.\n\n//Mr. Robot\n\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n"
# extract NCS tar
- bash ${FILE} --local-install nso-${NCS_VERSION}
# get list of tail-f issues
- pip3 install rt
- utils/tailf-rt.py --list-issues > tailf-issues
# prepare MR description based on tailf-issues and the changelog
- bash utils/prepare-mr-body.sh tailf-issues nso-${NCS_VERSION}/CHANGES > mr-description
# clone nso-ts
- git clone git@gitlab.dev.terastrm.net:TeraStream/nso-ts.git
- cd nso-ts
- git fetch origin
- git branch -r
- git checkout -b nso-${NCS_VERSION} origin/nso-${NCS_VERSION} || git checkout -b nso-${NCS_VERSION}
# update NCS version in relevant files
- sed -i Dockerfile -e "s/^\(FROM.*\):[0-9]\+\(\.[0-9]\+\)\+$/\1:${NCS_VERSION}/"
- sed -i Makefile -e "s/^NCS_VERSION=.*/NCS_VERSION=${NCS_VERSION}/"
- sed -i .gitlab-ci.yml -e "s/^\( NCS_VERSION:[ ]\).*/\1\"${NCS_VERSION}\"/"
# if we are the NCS version in use there will be no diff
- git diff
# only commit if there are changes on this branch
# if we in the checkout above checked out a remote branch, then it is
# probably identical to this one so there will be nothing to commit
- git diff --exit-code || git commit -a -m "Use NSO v${NCS_VERSION}"
- git rebase master
# only push if we are different from master
- git diff --exit-code master || git push --force origin nso-${NCS_VERSION}
- cd ..
- sleep 5 # let CI start up
- 'utils/open-mr.py --project TeraStream/nso-ts --source-branch nso-${NCS_VERSION} --title "WIP: Use NSO v${NCS_VERSION}" --description-file mr-description'
build-4.2.3:
<<: *build_def
variables:
NCS_VERSION: "4.2.3"
build-4.2.4:
<<: *build_def
variables:
NCS_VERSION: "4.2.4"
build-4.3.3:
<<: *build_def
variables:
NCS_VERSION: "4.3.3"
build-4.4.3:
<<: *build_def
variables:
NCS_VERSION: "4.4.3"
mr-robot-4.2.3:
<<: *mr-robot-def
variables:
NCS_VERSION: "4.2.3"
mr-robot-4.2.4:
<<: *mr-robot-def
variables:
NCS_VERSION: "4.2.4"
mr-robot-4.3.3:
<<: *mr-robot-def
variables:
NCS_VERSION: "4.3.3"
mr-robot-4.4.3:
<<: *mr-robot-def
variables:
NCS_VERSION: "4.4.3"
#!/usr/bin/env python3
import json
import os
import re
import requests
import urllib.parse
class OpenMr:
def __init__(self, project_name):
self.base_url = "https://gitlab.dev.terastrm.net"
self.api_url = "https://gitlab.dev.terastrm.net/api/v4"
self.token = os.getenv("PRIVATE_TOKEN")
self.headers = { 'PRIVATE-TOKEN': self.token }
self.project_name = project_name
self.project_id = self.get_project_id(self.project_name)
self.project_url = "/projects/{}".format(self.project_id)
def path_guess(self, path):
""" Guess what the URL should be based on an input path
"""
if re.match('[A-Za-z]+://', path):
url = path
else:
url = "{}{}".format(self.api_url, path)
return url
def http_get(self, path):
""" Simple HTTP get on path relative to base_url and doing
authentication using our token
"""
url = self.path_guess(path)
return requests.get(url, headers=self.headers)
def http_post(self, path, data):
""" Simple HTTP get on path relative to base_url and doing
authentication using our token
"""
url = self.path_guess(path)
return requests.post(url, headers=self.headers, data=data)
def get_json(self, path):
""" Just do a HTTP get, assume it goes fine, and return the data as a
JSON struct
"""
r = self.http_get(path)
data = json.loads(r.text)
return data
def urlencode(self, text):
""" URL encode a string
"""
return urllib.parse.urlencode({'a': text})[2:]
def get_project_id(self, name):
""" Get GitLab project ID based on project path, like 'TeraStream/nso-ts'
"""
return int(self.get_json("/projects/{}".format(self.urlencode(name)))['id'])
def get_mr(self, mrid):
data = self.get_json(self.project_url + "/merge_requests/{}".format(mrid))
def create_mr(self, source_branch, target_branch, title, description):
data = {
'id': self.project_id,
'source_branch': source_branch,
'target_branch': target_branch,
'remove_source_branch': True,
'title': title,
'description': description
}
self.http_post(self.project_url + "/merge_requests", data=data)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--project', required=True, help="Project path to retry jobs for, e.g. 'TeraStream/nso-ts'")
parser.add_argument('--title', required=True, help="Title of MR")
parser.add_argument('--description-file', required=True, help="File containing MR description")
parser.add_argument('--source-branch', required=True, help="Source branch")
# TODO: could find out what the default branch is for the repo
parser.add_argument('--target-branch', default='master', help="Target branch")
args = parser.parse_args()
f = open(args.description_file, encoding='utf-8')
desc = f.read()
f.close()
mr = OpenMr(args.project)
if mr.get_mr(args.source_branch) is None:
mr.create_mr(args.source_branch, args.target_branch, args.title, desc)
#!/usr/bin/env bash
# This is our first script to open an MR. I've since written a open-mr.py - try to use it instead!
#
# Extract the host where the server is running, and add the URL to the APIs
[[ $HOST =~ ^https?://[^/]+ ]] && HOST="${BASH_REMATCH[0]}/api/v4/projects/"
CI_PROJECT_NAME_URLENCODED=$(echo ${CI_PROJECT_NAME} | python3 -c "import sys, urllib.parse; print(urllib.parse.urlencode({'a': sys.stdin.read().strip()})[2:])")
CI_PROJECT_ID=$(curl --silent --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}" ${HOST}${CI_PROJECT_NAME_URLENCODED} | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
# Look which is the default branch
TARGET_BRANCH=$(curl --silent "${HOST}${CI_PROJECT_ID}" --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}" | python3 -c "import sys, json; print(json.load(sys.stdin)['default_branch'])")
# The description of our new MR, we want to remove the branch after the MR has
# been closed
BODY="{
\"id\": ${CI_PROJECT_ID},
\"source_branch\": \"${CI_COMMIT_REF_NAME}\",
\"target_branch\": \"${TARGET_BRANCH}\",
\"remove_source_branch\": true,
\"title\": \"Robot: ${CI_COMMIT_REF_NAME}\",
\"assignee_id\":\"${GITLAB_USER_ID}\"
}";
# Require a list of all the merge request and take a look if there is already
# one with the same source branch
LISTMR=`curl --silent "${HOST}${CI_PROJECT_ID}/merge_requests?state=opened" --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}"`;
COUNTBRANCHES=`echo ${LISTMR} | grep -o "\"source_branch\":\"${CI_COMMIT_REF_NAME}\"" | wc -l`;
# No MR found, let's create a new one
if [ ${COUNTBRANCHES} -eq "0" ]; then
curl -X POST "${HOST}${CI_PROJECT_ID}/merge_requests" \
--header "PRIVATE-TOKEN:${PRIVATE_TOKEN}" \
--header "Content-Type: application/json" \
--data "${BODY}";
echo "Opened a new merge request: ${CI_COMMIT_REF_NAME} and assigned to you";
exit;
fi
echo "No new merge request opened";
#!/bin/bash
# this takes a list of Tail-F issues and converts it into a set of awk commands
# that can grep in the NCS changelog based on the issues but instead of just
# returning a single line it extracts the paragraph that mentions the issue.
for I in $(cat $1); do
echo "awk -v RS=' - ' '/${I}/ { print \" -\",\$0 }' $2";
done > changelog-grepper.sh
echo -e "According to the changelog, the following issues have been fixed:"
grep -f $1 $2 | sed -e 's/[^0-9]\+\([0-9][0-9][0-9]\+\)[^0-9]/ * https:\/\/support.tail-f.com\/rt\/Ticket\/Display.html?id=\1\n/g' | sed '/^$/d' | sort -n
echo -e "\nWith the following descriptions:\n\`\`\`"
bash changelog-grepper.sh
echo -e "\`\`\`\n\nFull changelog:\n\`\`\`"
cat $2
echo "\`\`\`"
#!/usr/bin/env python3
import os
import rt
username = os.getenv('TAILF_RT_USER')
password = os.getenv('TAILF_RT_PASS')
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--list-issues', action='store_true')
args = parser.parse_args()
tracker = rt.Rt('https://support.tail-f.com/rt/REST/1.0/', username, password)
tracker.login()
if args.list_issues:
# XXX change the name of the queue to your queue!
for ticket in tracker.search(Queue='DTAG', Status='open'):
print(ticket['id'].split('/')[1])
tracker.logout()
@tonyfahrion

This comment has been minimized.

Copy link

tonyfahrion commented Nov 9, 2017

Hi,

very good work! And it was a pleasure to read the related article at gitlab 😄

Just as a note, there seems to be a good and sufficient tool for automating the "merge request creation": https://github.com/NARKOZ/gitlab

Kind regards
Tony

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.