Python3 GraphQl cript to transfer issues

transferissues.py

# Copyright (c) 2021, Lawrence Livermore National Security, LLC
#
# Python3 script using GraphQL interface to GitHub to transfer
# issues between GitHub repos.
#
# Programmer: Mark C. Miller, Tue Jul 20 10:21:40 PDT 2021
#
import datetime, email.header, glob, mailbox, os, pytz
import re, requests, shutil, sys, textwrap, time
from difflib import SequenceMatcher

 
#
# Capture failure details to a continuously appending file
#
def captureGraphQlFailureDetails(gqlQueryName, gqlQueryString, gqlResultString):
    with open("email2discussions-failures-log.txt", 'a') as f:
        f.write("%s - %s\n"%(datetime.datetime.now().strftime('%y%b%d %I:%M:%S'),gqlQueryName))
        f.write("--------------------------------------------------------------------------\n")
        f.write(gqlResultString)
        f.write("\n")
        f.write("--------------------------------------------------------------------------\n")
        f.write(gqlQueryString)
        f.write("\n")
        f.write("--------------------------------------------------------------------------\n\n\n\n")

#
# Read token from 'ghToken.txt'
#
def GetGHToken():
    if not hasattr(GetGHToken, 'ghToken'):
        try:
            with open('ghToken.txt', 'r') as f:
                GetGHToken.ghToken = f.readline().strip()
        except:
            raise RuntimeError('Put a GitHub token in \'ghToken.txt\' readable only by you.')
    return GetGHToken.ghToken

#
# Build standard header for URL queries
#
headers = \
{
    'Content-Type': 'application/json',
    'Authorization': 'bearer %s'%GetGHToken(),
    'GraphQL-Features': 'discussions_api'
}

#
# Workhorse routine for performing a GraphQL query
#
def run_query(query): # A simple function to use requests.post to make the API call. Note the json= section.

    if not hasattr(run_query, 'numSuccessiveFailures'):
        run_query.numSuccessiveFailures = 0;

    # Post the request. Time it and keep 100 most recent times in a queue
    try:
        request = requests.post('https://api.github.com/graphql', json={'query': query}, headers=headers)
        result = request.json()
        run_query.numSuccessiveFailures = 0

    except:

        captureGraphQlFailureDetails('run_query', query, "")
        run_query.numSuccessiveFailures += 1
        if run_query.numSuccessiveFailures > 3:
            raise Exception(">3 successive query failures, exiting...")
            sys.exit(1)

    if request.status_code == 200:
        return request.json()
    else:
        raise Exception("run_query failed with code of {}. {} {}".format(request.status_code, query, request.json()))

#
# A method to periodically call to ensure we don't
# exceed GitHub's rate limits
#
def throttleRate():

    # set the *last* check 61 seconds in the past to force a check
    # the very *first* time we run this
    if not hasattr(throttleRate, 'lastCheckNow'):
        throttleRate.lastCheckNow = datetime.datetime.now()-datetime.timedelta(seconds=61)

    query = """
        query
        {
            viewer
            {
                login
            }
            rateLimit
            {
                limit
                remaining
                resetAt
            }
        }
    """

    # Perform this check only about once a minute
    now = datetime.datetime.now()
    if (now - throttleRate.lastCheckNow).total_seconds() < 60:
        return
    throttleRate.lastCheckNow = now

    try:
        result = run_query(query)

        zuluOffset = 7 * 3600 # subtract PDT timezone offset from Zulu
        
        if 'errors' in result.keys():
            toSleep = (throttleRate.resetAt-now).total_seconds() - zuluOffset + 1
            print("Reached end of available queries for this cycle. Sleeping %g seconds..."%toSleep)
            time.sleep(toSleep)
            return

        # Gather rate limit info from the query result
        limit = result['data']['rateLimit']['limit']
        remaining = result['data']['rateLimit']['remaining']
        # resetAt is given in Zulu (UTC-Epoch) time
        resetAt = datetime.datetime.strptime(result['data']['rateLimit']['resetAt'],'%Y-%m-%dT%H:%M:%SZ')
        toSleep = (resetAt-now).total_seconds() - zuluOffset
        print("GraphQl Throttle: limit=%d, remaining=%d, resetAt=%g seconds"%(limit, remaining, toSleep))

        # Capture the first valid resetAt point in the future
        throttleRate.resetAt = resetAt

        if remaining < 200:
            print("Reaching end of available queries for this cycle. Sleeping %g seconds..."%toSleep)
            time.sleep(toSleep)

    except:
        captureGraphQlFailureDetails('rateLimit', query, "")

#
# Get various visit-dav org. repo ids. Caches results so that subsequent
# queries don't do any graphql work.
#
def GetRepoID(orgname, reponame):
    query = """
        query
        {
            repository(owner: \"%s\", name: \"%s\")
            {
                id
            }
        }
    """%(orgname, reponame)
    if not hasattr(GetRepoID, reponame):
        result = run_query(query)
        # result = {'data': {'repository': {'id': 'MDEwOlJlcG9zaXRvcnkzMjM0MDQ1OTA='}}}
        setattr(GetRepoID, reponame, result['data']['repository']['id'])
    return getattr(GetRepoID, reponame)


#
# Get object id by name for given repo name and org/user name. 
# Caches reponame/objname pair so that subsequent queries don't do any
# graphql work.
#
def GetObjectIDByName(orgname, reponame, gqlObjname, gqlCount, objname):
    query = """
        query
        {
            repository(owner: \"%s\", name: \"%s\")
            {
                %s(first:%d)
                {
                    edges
                    {  
                        node
                        {
                            description,
                            id,
                            name
                        }
                    }
                } 
            }
        }
    """%(orgname, reponame, gqlObjname, gqlCount)
    if not hasattr(GetObjectIDByName, "%s.%s"%(reponame,objname)):
        result = run_query(query)
        # result = d['data']['repository']['discussionCategories']['edges'][0] =
        edges = result['data']['repository'][gqlObjname]['edges']
        for e in edges:
            if e['node']['name'] == objname:
                setattr(GetObjectIDByName, "%s.%s"%(reponame,objname), e['node']['id'])
                break
    return getattr(GetObjectIDByName, "%s.%s"%(reponame,objname))


def getIssueIDByNumber(orgname, reponame, issuenum):
    query = """
        query
        {
            repository(owner: \"%s\", name: \"%s\")
            {
                issue(number:%d)
                {
                    id,
                    title
                }
            }
        }
    """%(orgname, reponame, issuenum)
    try:
        result = run_query(query)

        if 'errors' in result and len(result['errors']) and \
           'type' in result['errors'][0]:
            if result['errors'][0]['type'] == 'NOT_FOUND':
                return False
        else:
            return result['data']['repository']['issue']['id']
    except:
        captureGraphQlFailureDetails('issue (by-number) %d'%issuenum, query,
            repr(result) if 'result' in locals() else "")
    
    return None

#
# lock an object (primarily to lock a discussion)
#
def lockLockable(nodeid):
    query = """
        mutation
        {
            lockLockable(input:
            {
                clientMutationId:\"scratlantis:emai2discussions.py\",
                lockReason:RESOLVED,
                lockableId:\"%s\"
            })
            {
                lockedRecord
                {
                    locked
                }
            }
        }"""%nodeid
    try:
        result = run_query(query)
    except:
        captureGraphQlFailureDetails('lockLockable %s'%nodeid, query,
            repr(result) if 'result' in locals() else "")

#
# Add a convenience label to each discussion
# The label id was captured during startup
#
def addLabelsToLabelable(nodeid, labid):
    query = """
        mutation
        {
            addLabelsToLabelable(input:
            {
                clientMutationId:\"scratlantis:emai2discussions.py\",
                labelIds:[\"%s\"],
                labelableId:\"%s\"
            })
            {
                labelable
                {
                    labels(first:1)
                    {
                        edges
                        {  
                            node
                            {
                                id
                            }
                        }
                    }
                }
            }
        }"""%(labid, nodeid)
    try:
        result = run_query(query)
    except:
        captureGraphQlFailureDetails('addLabelsToLabelable %s'%nodeid, query,
            repr(result) if 'result' in locals() else "")

def transferIssue(issueid, repoid):
    query = """
        mutation
        {
            transferIssue(input:
            {
                issueId:\"%s\",
                repositoryId:\"%s\"
            })
            {
                issue
                {
                    id
                }
            }
        }"""%(issueid, repoid)
    try:
        result = run_query(query)
        return result['data']['transferIssue']['issue']['id']
        
    except:
        captureGraphQlFailureDetails('transferIssue %s'%issueid, query,
            repr(result) if 'result' in locals() else "")


#
# Main Program
#
# Get the repository id where the discussions will be created
dstrepoid = GetRepoID("visit-dav", "visit")

# Get the label id for the 'visit-uers email'
labid =  GetObjectIDByName("visit-dav", "visit", "labels", 30, "sre")

for i in range(350):

    print("Working on issue %d"%i)
    if i%20 == 0:
        throttleRate()
    issueid = getIssueIDByNumber("visit-dav", "live-customer-response", i)
    if issueid:
        newid = transferIssue(issueid, dstrepoid)
        print("Completed transfer")
        addLabelsToLabelable(newid, labid)