grpbibedit.py

#!/usr/bin/python
##########################################################
## grpbibedit.py
##    pulls data down from RT
##    2013-05-11  created by BH
##    2014        adapted by MS
##    2015-06-16  updated by BH
##
## hosted at:
##    test: https://tislnx3.slac.stanford.edu/cgi-bin/testgrpbibedit.py
##    prod: https://tislnx3.slac.stanford.edu/cgi-bin/grpbibedit.py
##
## dashboards created at, ex.,
##    https://tislnx3.slac.stanford.edu/mhuangbibedit.html
##
##########################################################
####TODO
## 1 find/create another RT account to use
## 3 rewrite with cookielib (per example)
## 4 check out python-rt https://git.nic.cz/redmine/projects/python-rt

import urllib, urllib2, datetime;  ## base64, json
import cgi, cgitb
cgitb.enable()

## don't know why it's submitting jobs as Sul.  
#UX  = 'cli'
#PX = 'rt_cli_pw'
CLAIM_LOG = '../log/RT_ticket_claim.log'
HISTORY_URL =  'https://inspirert.cern.ch/REST/1.0/ticket' #/<ticket-id>/history?format=l'

## user and pass parms supported
## http://www.gossamer-threads.com/lists/rt/users/75160
#base64string = base64.encodestring('%s:%s' % (ux, px)).replace('\n', '')


############################################################################

def logThis(msg):

	LOGFILE = open(CLAIM_LOG,'a')
	LOGFILE.write(str(datetime.datetime.now())+'\t' + msg + '\n')
	LOGFILE.close()
#############################################################################

def build_form_body( ):

	formbod = '''
	<form name="input" action="grpbibedit.py" method="get">
	<hr/>

	<div>Select your RT account:<select name="cataloger">
		<option value="mhuang">mhuang</option>
		<option value="kornienk">kornienk</option>
		<option value="sul">sul</option>
		<option value="bhecker">bhecker</option>
		<option value="thorsten">thorsten</option>
	        <option selected value="none">none</option>
	</select>
        RT Password:<input type="password" name="pwd" required>
	Queue: <select name="queue">
           <option selected value="HEP_Curation">HEP_Curation</option>
            <option value="Inspire-References">Inspire-References</option>
        </select>
	</div>

	<div>Select the number of tickets you want:<select name="quantity">
	<option value="5">5</option>
	<option value="25">25</option>
	<option value="50" selected>50</option>
	</select></div>

	<input type="submit" value="Submit">
	</form>

	'''

	return( formbod )


###################################################################
def send_request(req_url):
	try:
		request = urllib2.Request(req_url)
	        #request.add_header("Authorization", "Basic %s" % base64string)   
		outx =  urllib2.urlopen(request)
		fromRT = outx.read()
		outx.close()
	except urllib2.URLError:
		# no server connection
		fromRT = "Could not connect to server."

	if "Credentials required" in fromRT:
		fromRT = 'Error. Credentials required.'
	
	return( fromRT )

#############################################################################
def get_available_tickets(queuex, catx, passx):

	# query RT for a sorted list of tickets in queue QUEUE that are
	# owned by "Nobody" and are status="new"

	reqdatx = {}
	reqdatx['user'] = catx
	reqdatx['pass'] = passx
	reqdatx['query'] = "Queue='" + queuex + "' AND Owner='Nobody' AND status='new'"
	reqdatx['orderby'] = '+Created' ; ## ordering; use '-' for descending
	reqdatx['format'] = 'l'

	base_url= 'https://inspirert.cern.ch/REST/1.0/search/ticket'

	# encode parameter list, build request URL
	req_url = base_url + '?' + urllib.urlencode(reqdatx)

	result = send_request(req_url)
	return (result)

#############################################################################

def claim_tix(available_tix_result, quantx, catx, passx):
	header = True 
	rx = []
	xmsg = ''

	if ( available_tix_result == "Could not connect to server.") | (available_tix_result == 'Error. Credentials required.') :
		xmsg += "<h3>ERROR: %s </h3>" % available_tix_result
		logThis("REQUEST ERROR:"+ available_tix_result)
	else:
		#xmsg += "<p>Starting parse..."+str( datetime.datetime.now())+'</p>'

		## parse the result returned by the query for available tickets, harvest RT IDs  ##########
		for r in available_tix_result.split('\n'):

			if ("id:" in r) | ("Created:" in r) | ("Subject" in r):
				(label, value) = r.split(': ',1)
				if "id" in label:
					# it's a new ticket, so store the values from the last ticket
					# unless it's the very first record found
					if header: header = False
					else:
						rx.append( {'id':current_id, 'created':current_created, 'subject':current_subject } )
					ticket_prefix, current_id = value.split('ticket/')
				elif "Created" in label:
					current_created = value
				elif "Subject" in label:
					current_subject = value
		#xmsg += "<p>Finished parse..." + str(datetime.datetime.now()) + "  # of tickets:" + str(len(rx))+'</p>'
		
		selected_ticket_ids = []
		logThis("Requesting ownership of "+str(quantx)+" tickets for "+ catx +'...')
		
		### take ownership of QUANTITY tickets##############################################
		reqdatx = {}
		reqdatx['user'] = catx
		reqdatx['pass'] = passx
		reqdatx['content'] = 'Owner: '+ catx
		reqdatx['type'] = 'force'
		
		for i in range(0, min(quantx, len(rx))):
			startTime= datetime.datetime.now()
			# xmsg += "<p>  ===> "+rx[i]['id']+"   "+rx[i]['created']+'</p>'

			base_url= 'https://inspirert.cern.ch/REST/1.0/ticket/'+str(rx[i]['id'])+'/edit'
			req_url = base_url + '?' + urllib.urlencode(reqdatx)

			# change the owner of the selected tickets
			status = send_request(req_url)

			if status == "Could not connect to server.":
				xmsg += "<h3>ERROR: Could not connect to server.</h3>"
			else:
				xmsg += '<code class="status">'+status+'</code>'
				endTime = datetime.datetime.now()
				tx = endTime - startTime
				xmsg += "<p>Duration:"+str(tx.seconds + tx.microseconds/1000000.0)+" seconds</p>"

				selected_ticket_ids.append(str(rx[i]['id']))

			xmsg += "."


		logThis("Ownership claimed for " + catx +'\t'+str(selected_ticket_ids)+'\n')

	##xmsg has durations for the claim operation
	return(xmsg, selected_ticket_ids)


#############################################################################
def buildLinkout(InspireIDs, GROUP, catx, passx):

	bibeditHead = '<a href="https://inspirehep.net/record/edit/?ln=en#state=search&p='
        bibeditHTML = ''

        bibeditLink = ''
        recidlist = ''
	#logThis("InspireIDs:" + str(InspireIDs))
	if InspireIDs == []:
		bibeditlink = ''
		recidlist = ''
	else:
		grouplink = ''
		for i in range(0,len(InspireIDs),GROUP):
			grouplink =  'recid:'+'%20or%20recid:'.join(InspireIDs[i:i+GROUP])
			bibeditLink += '<li>'+bibeditHead + grouplink + '" target=bibeditmult>Link to BibEdit ' +str(i)+ '</a>' + '</li>\n'

		datex = datetime.datetime.now().strftime("%Y:%m:%d  %H:%M:%S")
                bibeditHTML = '<html><head><title>Dashboard linkouts</title></head><body>\n' +'<h2>' + CATALOGER \
			      + ':  Dashboard for Queue ' + QUEUE + '</h2>\n <ol>\n'+ bibeditLink \
			      + '</ol>\n\n <code>'+ datex + '</code></body></html>'
                slacfile = '/var/www/html/' + CATALOGER + 'bibedit.html'
                outfile = open(slacfile,'wb')
                outfile.write(bibeditHTML)
                outfile.close()

		## no idea what this file is used for
		for c in range(0, len(InspireIDs)):
			recidlist += str(InspireIDs[c]) + '\n'
		with open('/tmp/recidlist.txt','wb') as outfile:
			outfile.write(recidlist)

	return( bibeditHTML )

#############################################################################

def getInspireIDs(tix, catx, passx):
	### get the INSPIRE record IDs associated with the tickets returned
	
	ticArc = list()
	logThis("getInspireID request dispatch loop...")

	### one query per InspireID ######################
	for t in tix:
		reqdatx = {}
		reqdatx['user'] = catx
		reqdatx['pass'] = passx
		req_url = HISTORY_URL + '/'+str(t)+'/history?format=l&' + urllib.urlencode(reqdatx)
		result = send_request(req_url)
		## I think this "result" has all of the **history** on the ticket, i.e., every
		## transaction over time. That's why it's so long.
		#logThis("InspireID query result: "+result)

		for r in result.split('\n'):

			# this catchs both null lines, and lines that have only whitespace
			if r.lstrip() != '':
				# next, skip lines included from previous comments
				if (r.lstrip()[0] != ">"):
					if ("recid=" in r):
						prefix, arxnum = r.split('recid=')

						###  some recid result lines end in '.' , so strip it ###
						ticArc.append(arxnum.rstrip('.'))

	logThis("getInspireID request completed...")

	if ticArc == (): ticArc =('NO_MATCHES_FOUND')

	return(ticArc) 

#############################################################################
# main
# http://docs.python.org/2/howto/urllib2.html

# read any parameters
xform = cgi.FieldStorage()
formmsg = ''

GROUPBY = 5  # number of tickets to include in each linkout

## check to see if a form has been submitted
if xform.has_key('cataloger'):
	HaveParm = True
	CATALOGER = xform.getvalue("cataloger")
	PASSWORD = xform.getvalue('pwd') 
	QUANTITY = int(xform.getvalue('quantity'))
        QUEUE = xform.getvalue('queue')
	formmsg = "<p>cataloger:" +  CATALOGER + ' Queue: ' + QUEUE + '</p>\n'
	formmsg += "<p>quantity:" +  str(QUANTITY)  + '</p>\n\n</hr>'
else:
	# "no parameters submitted = no form submitted"
	HaveParm = False


htmlheader = '''Content-Type: text/html\n\n'''

htmltop = '''<html><head>
	<title>RT ticket batch claim</title></head>
	<body>
	<h1>RT batch ticket claiming for catalogers</h1>
	<h5>v 2.0</h5>
	<p>Please select your account and the number of tickets, and then submit. 
'''
htmltail = '</body></html>'



if HaveParm:

	logThis("+================= Run beginning ================+")
	logThis(CATALOGER + ", queue=" + QUEUE + ", quantity=" + str(QUANTITY) ) 


	reqdata = {}
	reqdata['user'] = CATALOGER
	reqdata['pass'] = PASSWORD
	reqdata['query'] = "Queue='" + QUEUE + "' AND Owner='Nobody' AND status='new'"
	reqdata['orderby'] = '+Created' ; ## ordering; use '-' for descending
	reqdata['format'] = 'l'

	# execute the query to pull oldest available tickets in QUEUE owned by Nobody
	available_tix_result = get_available_tickets(QUEUE, CATALOGER, PASSWORD)
	
	logThis("Tickets pulled....")

	# break the result into an array of dictionaries  { id, created, subject }
	#  then claim QUANTITY tickets, and log the result
	claim_msg, selected_tix = claim_tix(available_tix_result, QUANTITY, CATALOGER, PASSWORD)

	# get the INSPIRE IDs that correspond to those RT records

	InspireIDlist = getInspireIDs(selected_tix, CATALOGER, PASSWORD)

	logThis("INSPIRE IDs:"+str(InspireIDlist) )
	
	# return the bibedit linkout markup, 
	linktext = buildLinkout(InspireIDlist, GROUPBY, CATALOGER, PASSWORD)

	
	if InspireIDlist[0] == []:
		linktext = ''
		toDashboard = 'No RecIDs found in these tickets. No dashboard generated.'
	else:
                toDashboard  = 'Go to the dashboard page: <a href=https://inspire-vm1.slac.stanford.edu/'  + \
                CATALOGER + 'bibedit.html target=bibedit>' + CATALOGER + 'bibedit.html </a> \n'          

	
	finalmsg = htmlheader + htmltop + '\n  ' + '\n'+build_form_body() + '\n' + toDashboard + \
		'<hr/>\n<div class="inspire-ids"><h3>Inspire IDs</h3>\n<p>' + ", ".join(InspireIDlist) + '</p>\n\n' + \
		'<hr/></div>\n ' +  htmltail

	print finalmsg
	logThis("+----------------- Completed Run ----------------+")


else:
	finalmsg = htmlheader + htmltop + '\n  ' + '\n'+build_form_body() + '\n' \
		'<hr/>\n ' +  htmltail

	print finalmsg
	logThis("+--     --    -- - Form Displayed - --  --  --  -+")



#finalmsg = htmlheader + htmltop + '\n  ' + parmwarn + '\n'+build_form_body() + '\n' + linktext + correspCopy + \
#	correspTitle + InspireIDlist + '\n\n' + \
#	'<hr/>\n <em>' + claim_msg + '</em>\n' +  htmltail