Sample code to support a few Qxf2 blog posts about analyzing JIRA data using Python. In this gist, we show you the JiraDatastore.py module which saves individual ticket information as a pickled file.

JiraDatastore.py

"""
A class that stores JIRA ticket data as pickled files

DISCLAIMER: This is sample code written by Qxf2 Services to support a blog post about analyzing JIRA data using Python
This code does not represent Qxf2 Services's coding standards.
"""
from jira import JIRA
from ConnectJira import ConnectJira
import credentials as secret
import json
import pickle
import os, sys
import arrow
import datetime

class JiraDatastore:
    def __init__(self,server,username,password,project_name):
        self.connect_jira_obj = ConnectJira(server,username,password,project_name)
        self.project = project_name
        self.server = server
    
    def get_file_dir(self):
        "make file dir path with client and project params"
        client = (self.server).split('.')
        client = client[0].split('//')[-1]
        client = client.split(':')[0]

        file_dir = os.path.join('.','data',client.replace(' ','_'),self.project.replace(' ','_'))

        return file_dir

    def get_ticket_list(self,jql):
        "Return jira ticket list based on jql"
        jql_result = self.connect_jira_obj.execute_query(jql)
    
        return jql_result

    def get_last_updated_date(self):
        "get datastore last updated date"
        last_updated_date = None
        try:
            file_path = os.path.join(self.get_file_dir(),'datastore_updated_date.pickle')
            last_updated_date = self.pickle_in(file_path)
            last_updated_date = last_updated_date.strftime('%Y/%m/%d %H:%M:%S')
        except IOError:
            print "could not write to file: %s" %(file)
        
        return last_updated_date

    def pickle_out(self,data,file):
        "store data using pickle"
        result = False
        try:
            pickle_out = open(file,"wb")
            pickle.dump(data, pickle_out)
            pickle_out.close()
            result = True
        except IOError:
            print "could not write to file: %s" %(file)
        
        return result

    def pickle_in(self,file):
        "read data from pickle file"
        data = None
        try:
            pickle_in = open(file,"rb")
            data = pickle.load(pickle_in)
        except IOError:
            print "could not read file: %s" %(file)
        
        return data
    
    def get_pickle_ticket_in_json(self,ticket):
        ticket_in_json, error = None, None
        try:
            #Do datastore update before fetching ticket from pickle db
            fpath = os.path.join(self.get_file_dir(), ticket + '.pickle')
            pickle_ticket = self.pickle_in(fpath)
            if pickle_ticket is not None:
                ticket_in_json = pickle_ticket
            else:
                result = self.connect_jira_obj.get_ticket_in_json(
                    ticket)
                jira_ticket_in_json = result['ticket']
                error = result['error']
                # print ticket, ticket_in_json
                'store new ticket into our pickle data'
                if jira_ticket_in_json is not None:
                    ticket_in_json = jira_ticket_in_json
                    result = datastore_obj_test.save_ticket(ticket, ticket_in_json)
                    if result is True:
                        print "save ticket successful"
                    else:
                        print "save ticket unsuccessful"
        except Exception as e:
            error = e
        
        return {'ticket_in_json':ticket_in_json, 'error':error}

    def get_item_in_json_ticket(self,ticket_in_json,item):
        item_in_json_ticket = None
        if type(ticket_in_json) is dict:
            issues = ticket_in_json.get('issues',None)
            if issues is not None:
                for issue in issues:
                    item_in_json_ticket = issue.get(item, None)
            else:
                print 'key:%s not found in the given json data'%(item)
        else:
            print 'Given ticket is not a dict/json'
        return item_in_json_ticket

    def save_ticket(self,ticket_key,ticket_expanded):
        "save ticket into pickle file"
        save_ticket = False
        file_name = '%s.pickle'%((ticket_key).strip())
        file_dir = self.get_file_dir()
        file_path = os.path.join(file_dir,file_name)
        print file_path
        if not os.path.exists(file_dir):
            os.makedirs(file_dir)
        save_ticket = self.pickle_out(ticket_expanded,file_path)

        return save_ticket

    def save_last_updated_date(self):
        "save datastore last updated date"
        save_last_updated_date = False
        datastore_updated_date = datetime.datetime.now()
        file_name = 'datastore_updated_date.pickle'
        file_dir = self.get_file_dir()
        file_path = os.path.join(file_dir,file_name)
        print 'datastore updated date: %s' %(datastore_updated_date)
        save_last_updated_date = self.pickle_out(datastore_updated_date,file_path)

        return save_last_updated_date

    def store_ticket_list(self,ticket_list):
        "run datastore using pickle for given jira ticket list"
        datastore_update_complete, save, error = False, False, None
        for index,ticket in enumerate(ticket_list):
            "need one more jira call to expand changelog and fields in json format"
            # If json is set to true , jira call can get only one result 
            # Above error shown when we try to execute search issues call for multiple tickets 
            # expand = changelog fetches fields also 
            result = self.connect_jira_obj.get_ticket_in_json(ticket.key)
            ticket_in_json = result['ticket']
            error = result['error']
            if ticket_in_json is not None:
                # Separate pickle file for each ticket 
                save = self.save_ticket(ticket.key,ticket_in_json)
            else:
                print 'Jira returned no json input for the ticket: %s, error: %s'%(ticket.key,error) 
        if save is True:
            if index == len(ticket_list)-1:
                print "stored pickled files for the given ticket list: %d"%(len(ticket_list))
                datastore_update_complete = self.save_last_updated_date()
        
        return datastore_update_complete
                
    def datastore(self):
        #set to single ticket id for test run
        no_of_updated_tickets = 0
        #ticket_id = 'PORTAL-1923'
        "This jql fetches only list of tickets"
        jql = "project='%s' ORDER BY updated DESC" % (self.project)
        jql_result = self.get_ticket_list(jql)
        ticket_list = jql_result['ticket_list']
        error = jql_result['error']
        if ticket_list is not None:
            datastore_update_complete = self.store_ticket_list(ticket_list)
            if datastore_update_complete == True:
                no_of_updated_tickets = ticket_list.total       
        
        return {'no_of_updated_tickets':no_of_updated_tickets,'error':error}

    def update_datastore(self):
        "update jira datastore based on last updated date"
        no_of_updated_tickets = 0
        last_updated_date = self.get_last_updated_date()
        if last_updated_date is not None:
            print 'last_updated_date: %s'%(last_updated_date)
            "This jql fetches only list of tickets"
            jql = "project = '%s' AND updatedDate >= '%s' ORDER BY updated DESC" % (self.project, last_updated_date)
            jql_result = self.get_ticket_list(jql)
            ticket_list = jql_result['ticket_list']
            error = jql_result['error']
            if ticket_list is not None:
                datastore_update_complete = self.store_ticket_list(ticket_list)
                if datastore_update_complete == True:
                    no_of_updated_tickets = ticket_list.total

            return {'no_of_updated_tickets':no_of_updated_tickets,'error':error}
        else: #if we have never updated the datastore before
            return self.datastore() #update from scratch
    
    def update_datastore_status(self):
      "Return datastore status"
      last_updated_date = self.get_last_updated_date()
      if last_updated_date is not None:
        jql = "project = '%s' AND updatedDate >= '%s' ORDER BY updated DESC" % (self.project, last_updated_date)
        update_status = "Datastore last updated date: %s, "%last_updated_date
      else:
        jql = "project='%s' ORDER BY updated DESC" % (self.project)
        update_status = "Datastore does not exist before, "
    
      jql_result = self.get_ticket_list(jql) 
      ticket_list = jql_result['ticket_list']
      error = jql_result['error']
      if ticket_list is not None:
        ticket_count = len(ticket_list)
        error = jql_result['error']
        if 0 < ticket_count < 50:
            update_status = update_status + "received %s tickets to be updated, datastore update may take hardly a minute. Please wait patiently."%ticket_count
        if 50 < ticket_count < 100:
            update_status = update_status + "received %s tickets to be updated, datastore update may take 1-2 minutes. Please wait patiently."%ticket_count
        if 100 < ticket_count < 200:
            update_status = update_status + "received %s tickets to be updated, datastore update may take few minutes. Please wait patiently."%ticket_count
        if 200 < ticket_count < 300:
            update_status = update_status + "received %s tickets to be updated, datastore update will take few minutes. Please wait patiently."%ticket_count 
        if 500 < ticket_count < 1000:
            update_status = update_status + "received %s tickets to be updated, datastore update may take around 10 - 15 minutes. Please wait patiently."%ticket_count
        if 1000 < ticket_count < 3000:
            update_status = update_status + "received %s tickets to be updated, datastore update may take around 20 - 30 minutes. Please wait patiently."%ticket_count 
        if ticket_count > 3000:
            update_status = update_status + "received %s tickets to be updated, datastore will take more than 30 minutes. Please wait patiently."%ticket_count 
        if ticket_count == 0:
            update_status = update_status + "received 0 tickets from Jira, hence no updates to datastore."
      else:
        update_status = update_status + "Received None ticket list from Jira, error: %s"%error

      return update_status


if __name__ == '__main__':
    import credentials as secret
    datastore_obj_test = JiraDatastore(secret.JIRA_URL, secret.USERNAME,
                                secret.PASSWORD, secret.PROJECT)
    print datastore_obj_test.get_pickle_ticket_in_json('PORTAL-000')
    result = datastore_obj_test.update_datastore()
    """
    no_of_tickets = result['no_of_updated_tickets']
    error = result['error']
    result = datastore_obj_test.update_datastore()
    error = result['error']
    no_of_updated_tickets = result['no_of_updated_tickets']
    if no_of_tickets>0:
        print "datastore copy successful for no of tickets: %d"%(no_of_tickets)
    else:
        print "datastore copy unsuccessful: %s"%(error)
    if no_of_updated_tickets>0:
        print "datastore update successful for no of updated tickets: %d"%(no_of_updated_tickets)
    else:
        print "datastore update unsuccessful: %s"%(error)
    """