Git automation stuff

.gitlab-ci.yml

image: gitlab.dev.terastrm.net:4567/terastream/cisco-nso/ci-cisco-nso:4.2.3

stages:
  - build
  - mr-robot

variables:
  NCS_VERSION: "4.2.3"
  DOCKER_REGISTRY: "gitlab.dev.terastrm.net:4567/terastream/cisco-nso"
  TARGET_REGISTRY: "repo.dev.terastrm.net:5000/"
  FILE: "nso-${NCS_VERSION}.linux.x86_64.installer.bin"

.build.def: &build_def
  stage: build
  tags:
    - docker
  script:
    - if [[ ${CI_COMMIT_REF_NAME} != 'master' ]]; then export ADDITIONAL_TAG=-${CI_COMMIT_REF_NAME}; fi
    - git lfs fetch -I ${FILE}
    - git lfs checkout ${FILE}
    - make build-image
    - make tag-image
    - docker login -u gitlab-ci-token -p ${CI_BUILD_TOKEN} ${CI_REGISTRY}
    - make push-image
    # only push master to external repo
    - if [[ ${CI_COMMIT_REF_NAME} == 'master' ]]; then make push-image; fi
  after_script:
    # fugly hack for git lfs problems in gitlab ci
    - rm .git/hooks/post-checkout || true

.mr-robot-def: &mr-robot-def
  stage: mr-robot
  tags:
    - docker
  only:
    - master@TeraStream/cisco-nso
  before_script:
    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
    - eval $(ssh-agent -s)
    - ssh-add <(echo "$GIT_SSH_PRIV_KEY")
    - git config --global user.email "kll@dev.terastrm.net"
    - git config --global user.name "Mr. Robot"
    - mkdir -p ~/.ssh
    - cat gitlab-known-hosts >> ~/.ssh/known_hosts
  script:
    - echo -e "\n\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\nHello, I'm Mr. Robot and I will assist you with boring git tasks.\n\nI will now try to clone TeraStream/nso-ts, modify the NSO version used and submit a MR.\n\n//Mr. Robot\n\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n"
    # extract NCS tar
    - bash ${FILE} --local-install nso-${NCS_VERSION}
    # get list of tail-f issues
    - pip3 install rt
    - utils/tailf-rt.py --list-issues > tailf-issues
    # prepare MR description based on tailf-issues and the changelog
    - bash utils/prepare-mr-body.sh tailf-issues nso-${NCS_VERSION}/CHANGES > mr-description
    # clone nso-ts
    - git clone git@gitlab.dev.terastrm.net:TeraStream/nso-ts.git
    - cd nso-ts
    - git fetch origin
    - git branch -r
    - git checkout -b nso-${NCS_VERSION} origin/nso-${NCS_VERSION} || git checkout -b nso-${NCS_VERSION}
    # update NCS version in relevant files
    - sed -i Dockerfile -e "s/^\(FROM.*\):[0-9]\+\(\.[0-9]\+\)\+$/\1:${NCS_VERSION}/"
    - sed -i Makefile -e "s/^NCS_VERSION=.*/NCS_VERSION=${NCS_VERSION}/"
    - sed -i .gitlab-ci.yml -e "s/^\(  NCS_VERSION:[ ]\).*/\1\"${NCS_VERSION}\"/"
    # if we are the NCS version in use there will be no diff
    - git diff
    # only commit if there are changes on this branch
    # if we in the checkout above checked out a remote branch, then it is
    # probably identical to this one so there will be nothing to commit
    - git diff --exit-code || git commit -a -m "Use NSO v${NCS_VERSION}"
    - git rebase master
    # only push if we are different from master
    - git diff --exit-code master || git push --force origin nso-${NCS_VERSION}
    - cd ..
    - sleep 5 # let CI start up
    - 'utils/open-mr.py --project TeraStream/nso-ts --source-branch nso-${NCS_VERSION} --title "WIP: Use NSO v${NCS_VERSION}" --description-file mr-description'


build-4.2.3:
  <<: *build_def
  variables:
    NCS_VERSION: "4.2.3"

build-4.2.4:
  <<: *build_def
  variables:
    NCS_VERSION: "4.2.4"

build-4.3.3:
  <<: *build_def
  variables:
    NCS_VERSION: "4.3.3"

build-4.4.3:
  <<: *build_def
  variables:
    NCS_VERSION: "4.4.3"


mr-robot-4.2.3:
  <<: *mr-robot-def
  variables:
    NCS_VERSION: "4.2.3"

mr-robot-4.2.4:
  <<: *mr-robot-def
  variables:
    NCS_VERSION: "4.2.4"

mr-robot-4.3.3:
  <<: *mr-robot-def
  variables:
    NCS_VERSION: "4.3.3"

mr-robot-4.4.3:
  <<: *mr-robot-def
  variables:
    NCS_VERSION: "4.4.3"

open-mr.py

#!/usr/bin/env python3

import json
import os
import re
import requests
import urllib.parse

class OpenMr:
    def __init__(self, project_name):
        self.base_url = "https://gitlab.dev.terastrm.net"
        self.api_url = "https://gitlab.dev.terastrm.net/api/v4"
        self.token = os.getenv("PRIVATE_TOKEN")
        self.headers = { 'PRIVATE-TOKEN': self.token }

        self.project_name = project_name
        self.project_id = self.get_project_id(self.project_name)
        self.project_url = "/projects/{}".format(self.project_id)

    def path_guess(self, path):
        """ Guess what the URL should be based on an input path
        """
        if re.match('[A-Za-z]+://', path):
            url = path
        else:
            url = "{}{}".format(self.api_url, path)
        return url

    def http_get(self, path):
        """ Simple HTTP get on path relative to base_url and doing
            authentication using our token
        """
        url = self.path_guess(path)
        return requests.get(url, headers=self.headers)

    def http_post(self, path, data):
        """ Simple HTTP get on path relative to base_url and doing
            authentication using our token
        """
        url = self.path_guess(path)
        return requests.post(url, headers=self.headers, data=data)

    def get_json(self, path):
        """ Just do a HTTP get, assume it goes fine, and return the data as a
            JSON struct
        """
        r = self.http_get(path)
        data = json.loads(r.text)
        return data

    def urlencode(self, text):
        """ URL encode a string
        """
        return urllib.parse.urlencode({'a': text})[2:]

    def get_project_id(self, name):
        """ Get GitLab project ID based on project path, like 'TeraStream/nso-ts'
        """
        return int(self.get_json("/projects/{}".format(self.urlencode(name)))['id'])

    def get_mr(self, mrid):
        data = self.get_json(self.project_url + "/merge_requests/{}".format(mrid))

    def create_mr(self, source_branch, target_branch, title, description):
        data = {
            'id': self.project_id,
            'source_branch': source_branch,
            'target_branch': target_branch,
            'remove_source_branch': True,
            'title': title,
            'description': description
        }
        self.http_post(self.project_url + "/merge_requests", data=data)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--project', required=True, help="Project path to retry jobs for, e.g. 'TeraStream/nso-ts'")
    parser.add_argument('--title', required=True, help="Title of MR")
    parser.add_argument('--description-file', required=True, help="File containing MR description")
    parser.add_argument('--source-branch', required=True, help="Source branch")
    # TODO: could find out what the default branch is for the repo
    parser.add_argument('--target-branch', default='master', help="Target branch")
    args = parser.parse_args()

    f = open(args.description_file, encoding='utf-8')
    desc = f.read()
    f.close()
    mr = OpenMr(args.project)
    if mr.get_mr(args.source_branch) is None:
        mr.create_mr(args.source_branch, args.target_branch, args.title, desc)

open-mr.sh

#!/usr/bin/env bash
# This is our first script to open an MR. I've since written a open-mr.py - try to use it instead!
#
# Extract the host where the server is running, and add the URL to the APIs
[[ $HOST =~ ^https?://[^/]+ ]] && HOST="${BASH_REMATCH[0]}/api/v4/projects/"

CI_PROJECT_NAME_URLENCODED=$(echo ${CI_PROJECT_NAME} | python3 -c "import sys, urllib.parse; print(urllib.parse.urlencode({'a': sys.stdin.read().strip()})[2:])")
CI_PROJECT_ID=$(curl --silent --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}" ${HOST}${CI_PROJECT_NAME_URLENCODED} | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")

# Look which is the default branch
TARGET_BRANCH=$(curl --silent "${HOST}${CI_PROJECT_ID}" --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}" | python3 -c "import sys, json; print(json.load(sys.stdin)['default_branch'])")

# The description of our new MR, we want to remove the branch after the MR has
# been closed
BODY="{
    \"id\": ${CI_PROJECT_ID},
    \"source_branch\": \"${CI_COMMIT_REF_NAME}\",
    \"target_branch\": \"${TARGET_BRANCH}\",
    \"remove_source_branch\": true,
    \"title\": \"Robot: ${CI_COMMIT_REF_NAME}\",
    \"assignee_id\":\"${GITLAB_USER_ID}\"
}";

# Require a list of all the merge request and take a look if there is already
# one with the same source branch
LISTMR=`curl --silent "${HOST}${CI_PROJECT_ID}/merge_requests?state=opened" --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}"`;
COUNTBRANCHES=`echo ${LISTMR} | grep -o "\"source_branch\":\"${CI_COMMIT_REF_NAME}\"" | wc -l`;

# No MR found, let's create a new one
if [ ${COUNTBRANCHES} -eq "0" ]; then
    curl -X POST "${HOST}${CI_PROJECT_ID}/merge_requests" \
        --header "PRIVATE-TOKEN:${PRIVATE_TOKEN}" \
        --header "Content-Type: application/json" \
        --data "${BODY}";

    echo "Opened a new merge request: ${CI_COMMIT_REF_NAME} and assigned to you";
    exit;
fi

echo "No new merge request opened";

prepare-mr-body.sh

#!/bin/bash

# this takes a list of Tail-F issues and converts it into a set of awk commands
# that can grep in the NCS changelog based on the issues but instead of just
# returning a single line it extracts the paragraph that mentions the issue.

for I in $(cat $1); do
    echo "awk -v RS='  - ' '/${I}/ { print \"  -\",\$0 }' $2";
done > changelog-grepper.sh

echo -e "According to the changelog, the following issues have been fixed:"
grep -f $1 $2 | sed -e 's/[^0-9]\+\([0-9][0-9][0-9]\+\)[^0-9]/ * https:\/\/support.tail-f.com\/rt\/Ticket\/Display.html?id=\1\n/g' | sed '/^$/d' | sort -n
echo -e "\nWith the following descriptions:\n\`\`\`"
bash changelog-grepper.sh
echo -e "\`\`\`\n\nFull changelog:\n\`\`\`"
cat $2
echo "\`\`\`"

tailf-rt.py

#!/usr/bin/env python3

import os

import rt

username = os.getenv('TAILF_RT_USER')
password = os.getenv('TAILF_RT_PASS')


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--list-issues', action='store_true')
    args = parser.parse_args()

    tracker = rt.Rt('https://support.tail-f.com/rt/REST/1.0/', username, password)
    tracker.login()
    if args.list_issues:
        # XXX change the name of the queue to your queue!
        for ticket in tracker.search(Queue='DTAG', Status='open'):
            print(ticket['id'].split('/')[1])

    tracker.logout()

Hi,

very good work! And it was a pleasure to read the related article at gitlab 😄

Just as a note, there seems to be a good and sufficient tool for automating the "merge request creation": https://github.com/NARKOZ/gitlab

Kind regards
Tony