|
#!/bin/env python |
|
""" |
|
jenins-stats.py check the plugin usage in jenkins |
|
Usage: jenins-stats.py [options] |
|
|
|
Options: |
|
-d, --dir=CONFIG_DIR file directory to store or fetch config.xml |
|
-c, --config=JENKINS_CFG keywork mapping file, default is jenkins.cfg |
|
-u, --user=<your id> user to access jenkins |
|
|
|
-o, --token=xxx token for access user |
|
-p, --passwd enter password in stdin |
|
|
|
-l, --url=<JENKINS SERVER> start with http:// or https:// |
|
-t, --task=TASK plugins (default), scan, dump, jobs |
|
|
|
-h this help |
|
|
|
Examples: |
|
$ ./jenins-stats.py --task plugins --token xxxx --user rdccaiy --config_dir=tmp/jenkinsBackup --url https://jenkins_url |
|
$ ./jenins-stats.py --task plugins --passwd --user rdccaiy --config_dir=tmp/jenkinsBackup --url https://jenkins_url |
|
$ ./jenkins-stat.py --task plugins # list all plugins |
|
$ ./jenkins-stat.py --task jobs # list all jobs |
|
$ ./jenkins-stat.py --task dump --dir jenkinsBackup # dump all the jobs' config.xml |
|
$ ./jenkins-stat.py --task scan --dir jenkinsBackup # scan the config.xml and generate index.html |
|
|
|
Mail bug reports and suggestion to : Larry Cai |
|
""" |
|
|
|
import getopt, sys |
|
import os,time |
|
import logging |
|
import glob |
|
import codecs |
|
import getpass |
|
#import vcr |
|
try: |
|
import requests |
|
except ImportError: |
|
sys.stderr.write("\nError: please install python requests module first: http://www.python-requests.org/\n") |
|
|
|
from requests.auth import HTTPBasicAuth |
|
from string import Template |
|
import getpass |
|
import json |
|
|
|
currenttime = time.asctime(time.localtime()) |
|
|
|
LARRY_TOKEN = "a2310b1f6b7d584495ebbf9b347244e9fee" |
|
TEMPLATE_FILE = "plugin_usage_template.html" |
|
DETAIL_TEMPLATE_FILE = "detail_plugin_jobs_template.html" |
|
INDEX_HTML = "index.html" |
|
DETAIL_DIR = "detail" |
|
SHALL_DETAIL = True |
|
LIMIT_JOBS = 20 |
|
SYSTEM_VIEW_CFG = "views.config_xml" |
|
prop = { |
|
"user" : "larry", # will set to current user as default |
|
"token" : LARRY_TOKEN, |
|
"dir" : "jenkinsBackup", |
|
"config" : "jenkins.cfg", |
|
"passwd" : False, |
|
"url" : "https://jenkins-server/", |
|
} |
|
|
|
|
|
def get_job_config(url,name): |
|
config_url = "%s/job/%s/config.xml" % (url,name) |
|
|
|
#with vcr.use_cassette('fixtures/debug3.yaml'): |
|
# j = jenkins.Jenkins(url, 'rdccaiy', 'a231094d044ca829d367f35829085036') |
|
r = requests.get(config_url,verify=False,auth=HTTPBasicAuth(prop["user"], prop["token"])) |
|
if r.ok: |
|
return r.text |
|
else: |
|
raise Exception("can't fetch data from %s using user: %s, token: %s" % (config_url,prop["user"], prop["token"])) |
|
|
|
def get_jobs_from_view(url,view): |
|
|
|
jobs_url = "%s/view/%s/api/json" % (url, view) |
|
r = requests.get(jobs_url, verify=False,auth=HTTPBasicAuth(prop["user"], prop["token"])) |
|
if r.ok: |
|
return r.json() |
|
else: |
|
raise Exception("can't fetch data from %s using user: %s, token: %s" % (jobs_url,prop["user"], prop["token"])) |
|
|
|
def get_views_config(url): |
|
config_url = "%s/config.xml" % (url) |
|
|
|
#with vcr.use_cassette('fixtures/debug3.yaml'): |
|
# j = jenkins.Jenkins(url, 'rdccaiy', 'a231094d044ca829d367f35829085036') |
|
r = requests.get(config_url,verify=False,auth=HTTPBasicAuth(prop["user"], prop["token"])) |
|
if r.ok: |
|
return r.text |
|
else: |
|
raise Exception("can't fetch data from %s using user: %s, token: %s" % (config_url,prop["user"], prop["token"])) |
|
|
|
def get_plugins(url): |
|
plugin_url = "%s/pluginManager/api/json?depth=1" % url |
|
#with vcr.use_cassette('fixtures/debug_jobs.yaml'): |
|
r = requests.get(plugin_url, verify=False, auth=HTTPBasicAuth(prop["user"], prop["token"])) |
|
if r.ok: |
|
return r.json()["plugins"] |
|
else: |
|
raise Exception("can't fetch data from %s using user: %s, token: %s" % (plugin_url,prop["user"], prop["token"])) |
|
|
|
def get_jobs(url): |
|
jobs_url = "%s/api/json" % url |
|
r = requests.get(jobs_url, verify=False,auth=HTTPBasicAuth(prop["user"], prop["token"])) |
|
if r.ok: |
|
return r.json()["jobs"] |
|
else: |
|
raise Exception("can't fetch data from %s using user: %s, token: %s" % (jobs_url,prop["user"], prop["token"])) |
|
|
|
def list_plugins(url): |
|
print ("start to list installed plugins in url:", url) |
|
plugin_list = get_plugins(url) |
|
for plugin in plugin_list: |
|
#print ("%s (%s): version : %s") % (plugin["longName"],plugin["shortName"], plugin["version"]) |
|
print ("{0} ({1}): version : {2}".format(plugin["longName"],plugin["shortName"], plugin["version"])) |
|
# if plugin["shortName"] == "git": |
|
# print plugin |
|
#print ("total %d plugins") % len(plugin_list) |
|
print ("total %{0} plugins".format(len(plugin_list))) |
|
|
|
def list_jobs(url): |
|
print ("start to list all jobs in url:", url) |
|
job_list = get_jobs(url) |
|
for job in job_list: |
|
print (job["name"],job["color"]) #,job |
|
|
|
print ("\nTotal {0} jobs".format(len(job_list))) |
|
return job |
|
|
|
def load_plugin_keywords(): |
|
config = json.load(open(prop["config"])) |
|
plugins_config = {} |
|
for plugin in config["jobs"]["simple"]: |
|
plugins_config[plugin] = ["jobs",""] |
|
|
|
for plugin in config["views"]["simple"]: |
|
plugins_config[plugin] = ["views",""] |
|
|
|
for plugin in config["system"]: |
|
plugins_config[plugin] = ["system",""] |
|
#print plugins_config |
|
return plugins_config |
|
|
|
def check_jobs(url,name,key,config_files,plugin): |
|
count = 0 |
|
detail_html="" |
|
detail_table_data = "" |
|
|
|
for fl in config_files: |
|
#print fl |
|
# http://docs.python.org/2/howto/unicode.html |
|
if key in codecs.open(fl,encoding="utf-8",errors="ignore").read(): |
|
jobname = os.path.splitext(os.path.basename(fl))[0] |
|
|
|
count = count +1 |
|
if count < LIMIT_JOBS: |
|
job_url = "<a href='%s/job/%s/' target='_blank'>%s</a>, " % (url, jobname,jobname) |
|
detail_html += job_url |
|
elif count == LIMIT_JOBS: |
|
detail_html += "..." |
|
else: |
|
pass # don't append more |
|
if SHALL_DETAIL: |
|
detail_table_data += ' \ |
|
<tr class="gradeA"> \n \ |
|
<td><a href="%s">%s</a></td>\n \ |
|
<td class="center"><a href="%s/job/%s" target="_blank">%s</a></td>\n \ |
|
</tr>\n' % (plugin["url"],plugin["longName"],url,jobname,jobname) |
|
|
|
if count and SHALL_DETAIL: |
|
DETAIL_INDEX_HTML = "%s/%s.html" % (DETAIL_DIR,name) |
|
detailTemplateFilename = DETAIL_TEMPLATE_FILE |
|
detailTemplate = open(detailTemplateFilename).read() |
|
detail_html_contents = Template(detailTemplate).safe_substitute(detail_table_data=detail_table_data,currenttime=currenttime) |
|
|
|
with open(DETAIL_INDEX_HTML,"w") as fp: |
|
fp.writelines(detail_html_contents); |
|
detail_url = "0" |
|
if count: |
|
detail_url = '<a href="%s/%s.html" target="_blank">%d</a>' % (DETAIL_DIR,name,count) |
|
cell_data = '<td class="center hide" details="%s">%s</td>' % (detail_html,detail_url) |
|
|
|
jobs_url = "%s/api/json?pretty=true" % url |
|
|
|
type_data = '<td class="center"><a href="%s" title="list all jobs" target="_blank">jobs</a></td>' % jobs_url |
|
|
|
return count, cell_data, type_data |
|
|
|
def check_views(url,key): |
|
count = 0 |
|
config_dir = prop["dir"] |
|
config_url = "%s/config.xml" % url |
|
detail_url = "0" |
|
|
|
config_file = os.path.join(config_dir + "/" + SYSTEM_VIEW_CFG) |
|
if key in codecs.open(config_file,encoding="utf-8",errors="ignore").read(): |
|
count = 1 |
|
detail_url = '<a href="%s" target="_blank">%d</a>' % (config_url,count) |
|
|
|
cell_data = '<td class="center hide" details="">%s</td>' % (detail_url) |
|
type_data = '<td class="center"><a href="%s" title="list all views" target="_blank">views</a></td>' % config_url |
|
return count, cell_data, type_data |
|
|
|
def check_systems(): |
|
count = 1 |
|
cell_data = '<td class="center">1</td>' |
|
type_data = '<td class="center">system</td>' |
|
return count, cell_data, type_data |
|
|
|
def scan_plugins(url): |
|
config_dir = prop["dir"] |
|
config_files = glob.glob(config_dir + "/*.xml") |
|
|
|
print ("start to scan the config files for installed plugins") |
|
print ("1. start to check the plugins in url:", url) |
|
print ("2. scan {0} the config files under {1}".format(len(config_files),config_dir)) |
|
|
|
if len(config_files) == 0: |
|
print ("no config.xml are found, please dump first") |
|
return |
|
|
|
global_plugins = load_plugin_keywords() |
|
|
|
if SHALL_DETAIL: |
|
if not os.path.exists(DETAIL_DIR): |
|
print ("create dir: ", DETAIL_DIR) |
|
os.makedirs(DETAIL_DIR) |
|
|
|
plugin_list = get_plugins(url) |
|
total = len(plugin_list) |
|
|
|
table_data = "" |
|
for idx, plugin in enumerate(plugin_list): |
|
name = plugin["shortName"] |
|
#print ("plugin %3d/%d: %s (%s)") % (idx+1, total, plugin["longName"], name) , |
|
print ("plugin {0:3d}/{1}: {2} ({3})".format(idx+1, total, plugin["longName"], name)) |
|
|
|
# <org.jenkins.ci.plugins.html5__notifier.JobPropertyImpl plugin="html5-notifier-plugin@1.2"> |
|
plugin_cfg = ["",""] |
|
status = "?" |
|
if name in global_plugins: |
|
plugin_cfg = global_plugins[name] |
|
status = "=" |
|
|
|
plugin_type, key = plugin_cfg |
|
if plugin_type == "": |
|
plugin_type = "jobs" |
|
if key == "": |
|
key = 'plugin="' + name # |
|
else: |
|
status = "*" |
|
|
|
count = 0 |
|
cell_data = '<td class="center">0</td>' |
|
type_cell_data = '<td class="center">jobs</td>' |
|
if plugin_type == "system": |
|
count,number_cell_data,type_cell_data = check_systems() |
|
elif plugin_type == "views": |
|
count,number_cell_data,type_cell_data = check_views(url,key) |
|
elif plugin_type == "jobs": |
|
count,number_cell_data,type_cell_data = check_jobs(url,name,key,config_files,plugin) |
|
|
|
if count and status == "?": |
|
status = "=" |
|
|
|
if status == "?": |
|
type_cell_data = '<td title="need to add information in jenkins.cfg" class="center">unknown</td>' |
|
|
|
table_data += ' \ |
|
<tr class="gradeA"> \n \ |
|
<td><a href="%s">%s</a></td>\n \ |
|
<td class="shortname">%s</td>\n \ |
|
%s \n \ |
|
<td class="center">%s</td>\n \ |
|
%s \n \ |
|
</tr>\n' % (plugin["url"],plugin["longName"], name, number_cell_data, plugin["version"],type_cell_data) |
|
print ("found {0} jobs".format(count)) |
|
|
|
templateFilename = TEMPLATE_FILE |
|
template = open(templateFilename).read() |
|
html_contents = Template(template).safe_substitute(table_data=table_data,currenttime=currenttime) |
|
|
|
with open(INDEX_HTML,"w") as fp: |
|
fp.writelines(html_contents); |
|
|
|
print ("\nThe result is output to {0}, enjoy".format(INDEX_HTML)) |
|
|
|
if SHALL_DETAIL: |
|
print ("\nThe detail result is output to {0}, enjoy".format("detail")) |
|
|
|
def dump_config(url): |
|
dest_dir = prop["dir"] |
|
print ("start to dump all jobs' config.xml to {0} in url: {1}".format(dest_dir, url)) |
|
job_list = [] |
|
if "view" in prop: |
|
job_list = view_jobs_list(url) |
|
else: |
|
job_list = get_jobs(url) |
|
if not os.path.exists(dest_dir): |
|
print ("create dir: ", dest_dir) |
|
os.makedirs(dest_dir) |
|
total = len(job_list) |
|
print ("total {0} jobs' config will be dumpped".format(total)) |
|
|
|
for idx, job in enumerate(job_list): |
|
name = job["name"] |
|
config = get_job_config(url, name).encode('utf-8', 'ignore') |
|
output_file = os.path.normpath(os.path.join(dest_dir, name + ".xml")) |
|
|
|
#print (" %4d/%d dump output file %s") % (idx+1, total, output_file) |
|
print ("{0:4d}/{1} dump output file {2}".format(idx+1, total, output_file)) |
|
# loop for all variables to print out |
|
with open(output_file,"w") as fp: |
|
fp.write(str(config)) |
|
|
|
print ("total {0} jobs' config are dumpped".format(len(job_list))) |
|
|
|
config = get_views_config(url).encode('utf-8', 'ignore') |
|
output_file = os.path.normpath(os.path.join(dest_dir, SYSTEM_VIEW_CFG)) |
|
|
|
print ("views config.xml file is save to file {0}".format(output_file)) |
|
# loop for all variables to print out |
|
with open(output_file,"w") as fp: |
|
fp.write(str(config)) |
|
|
|
def get_jobs_list_from_view(url,view): |
|
print ("==> check view:", view) |
|
cfg = get_jobs_from_view(url,view) |
|
jobs = cfg["jobs"] |
|
if "views" in cfg: |
|
views = cfg["views"] |
|
print (view, ":", views) |
|
for vw in views: |
|
print (vw,vw["name"]) |
|
jobs.extend(get_jobs_list_from_view(url, view + "/view/" + vw["name"])) |
|
print (jobs) |
|
return jobs |
|
else: |
|
print ("======> got jobs list:", jobs) |
|
return jobs |
|
|
|
def view_jobs_list(url): |
|
view = prop["view"] |
|
job_list = get_jobs_list_from_view(url,view) |
|
print (job_list) |
|
#for job in job_list: |
|
# print job["name"],job["color"] #,job |
|
print ("\nTotal {0} jobs".format(len(job_list))) |
|
|
|
#http://stackoverflow.com/questions/11092511/python-list-of-unique-dictionaries |
|
unique_job_list = { v['name']:v for v in job_list }.values() |
|
print ("\nTotal {0} jobs".format(len(unique_job_list))) |
|
return unique_job_list |
|
#return job |
|
|
|
def jenkins_stat(task): |
|
url = prop["url"] |
|
|
|
#print ("[task]: %s %s (user=%s)") % (task,url,prop["user"]) |
|
print ("[task]: {0} {1} (user={2})".format(task,url,prop["user"])) |
|
|
|
if task == "plugins": |
|
list_plugins(url) |
|
elif task == "dump": |
|
dump_config(url) |
|
#config = get_job_config(url, "ADM_LABEL_SBA").encode('utf-8', 'ignore') |
|
#print config |
|
elif task == "jobs": |
|
list_jobs(url) |
|
elif task == "scan": |
|
scan_plugins(url) |
|
elif task == "debug": # this is internal for debug of hidden features |
|
view_jobs_list(url) |
|
else: |
|
print ("task", task, "is not supported") |
|
|
|
def main(): |
|
#logging.basicConfig(level=logging.DEBUG) |
|
global prop |
|
task = "plugins" |
|
prop["user"] = getpass.getuser().lower() |
|
|
|
try: |
|
cmdlineOptions, args= getopt.getopt(sys.argv[1:],'hu:a:t:o:c:d:l:pv:', |
|
["help","token=","task=","user=","output=","dir=","url=","config=","passwd","view="]) |
|
except getopt.GetoptError as e: |
|
print ("Error in a command-line option:\n\t" ,e) |
|
sys.exit(1) |
|
|
|
for (optName,optValue) in cmdlineOptions: |
|
if optName in ("-h","--help"): |
|
print (__doc__) |
|
sys.exit(1) |
|
elif optName in ("-d","--dir"): |
|
prop["dir"] = optValue |
|
elif optName in ("-c","--config"): |
|
prop["config"] = optValue |
|
elif optName in ("-t","--task"): |
|
task = optValue |
|
elif optName in ("-p","--passwd"): |
|
prop["passwd"] = True |
|
elif optName in ("-u","--user"): |
|
prop["user"] = optValue |
|
elif optName in ("-l","--url"): |
|
prop["url"] = optValue |
|
elif optName in ("-o","--token"): |
|
prop["token"] = optValue |
|
elif optName in ("-v","--view"): |
|
prop["view"] = optValue |
|
else: |
|
print ('Option %s not recognized' % optName) |
|
|
|
if prop["passwd"]: |
|
prop["token"] = getpass.getpass("Enter your ({0}) windows password: ".format(prop["user"])) |
|
elif prop["token"] == LARRY_TOKEN: |
|
print ("\n\t!!! Please use your own token to access jenkins server or use -p to enter your passwd !!! \n") |
|
|
|
jenkins_stat(task) |
|
|
|
if __name__ == "__main__": |
|
main() |
Hi Larry,can you explain how you are getting the complete list of used system plugins like Active directory etc? I do not understand from your script. If I look at the system's config.xml, it has few plugins in that, but its not a complete list. There are some plugins listed outside like for e.g hudson.plugins.timestamper.TimestamperConfig.xml etc. How do we find if these plugins are used or no? –
user1164061
comments from stackoverflowYes, it was incomplete. I just I add my interesting plugins now. I designed the
jenkins.cfg
to be extended with your own plugins, and pattern could be differed with different plugins, which may impact the script.I don't install the timestampler plugin, therefore it is not in the list. So far, there is no way to check whether it is view/job/system related, we have to check manually first.
You can check the jenkins GUI for
installed plugins
, then try to find it in different configuration.Feel free to fork and add your new things. --
Larry Cai