Skip to content

Instantly share code, notes, and snippets.

@jubel-han
Forked from marcelbirkner/ciSeedJob.groovy
Created November 4, 2016 03:34
Show Gist options
  • Save jubel-han/1d5236e5e54917d0613d243e23f26ca9 to your computer and use it in GitHub Desktop.
Save jubel-han/1d5236e5e54917d0613d243e23f26ca9 to your computer and use it in GitHub Desktop.
Jenkins Job DSL Seed Job for Continuous Integration Jobs
import groovy.sql.Sql
import java.util.Date
import java.text.SimpleDateFormat
/*
* THIS IS AN EXAMPLE SNIPPET. FOR MORE DETAILS SEE THE FOLLOWING BLOG ARTICLE:
* https://blog.codecentric.de/en/?p=30502
*
* This Jenkins Job DSL Groovy Script creates Continuous Integration (CI) Jobs
* for all Maven & Ant projects that exist on a GitLab Server.
*
* The script does the following steps:
* 1. Call GitLab REST API and get all projects
* 2. Call Configuration Management Database via JDBC and read additional information
* 3. Iterate over all projects and get further project details via GitLab REST API
* 4. Create all necessary jobs via Jenkins Job DSL. Typical jobs are: Build, Deploy, Acceptance Test-Jobs
* 5. Create custom views
*/
// GitLab settings
gitUrl = 'http://git/api/v3'
// you find the private token in your GitLab profile
gitPrivateToken = 'xYxYxYxYxYxYxYxYxYxY'
// id of Jenkins GitLab credentials
gitCredentials = "xYxYxYxY-xYxY-xYxY-xYxY-xYxYxYxYxYxY"
// mapping of Java Version to Jenkins JDK Version
jdkMap = ['6': 'JDK 6', '7': 'JDK 7', '8': 'JDK 8', '9': 'JDK 9']
javacMap = ['6': '1.6', '7': '1.7', '8': '1.8', '9': '1.9']
// Default build tool versions
def antVersion = "Ant 1.9.6"
def mavenVersion = "Maven 3.3.3"
// valid company department id's
departments = ["BIZ1","BIZ2","BIZ3","DEV1","DEV2","DEV3","OPS1","OPS2"]
// GitLab REST API settings. REST API returns max 100 per page. Thats why we need pagination.
def currentPage = 1
def projectsPerPage = 100
def currentProjectsSize = 100
// we will use this array to gather all CI job names
def ciJobList = []
// read projects from GitLab REST API until finished
while (currentProjectsSize == projectsPerPage) {
def projectsApi = new URL("${gitUrl}/projects/all?page=${currentPage}&per_page=${projectsPerPage}&private_token=${gitPrivateToken}")
println "############################################################################################################"
println "Read GitLab REST API: ${projectsApi}"
println "############################################################################################################"
// convert returned JSON to object
def projects = new groovy.json.JsonSlurper().parse(projectsApi.newReader())
// required for pagination
currentProjectsSize = projects.size()
currentPage++
// iterate over all projects
projects.each {
println "------------------------------------------------------------------------------------------------------------"
println "Working on project: ${it.name}"
def gitProjectName = it.name
def projectId = it.id
def projectGitSshUrlToRepo = it.ssh_url_to_repo
// read project details from configuration management databse
def projectDetails = getProjectDetailsFromConfigMgtDb(gitProjectName)
def isJavaProject=projectDetails.isJavaProject
if ( ! isJavaProject ) {
println "-> skipping project, since no Config Mgt details are available."
return
}
def webContextRoot=projectDetails.webContextRoot
def mailRecipients=projectDetails.mailRecipients
def isAnt=projectDetails.isAnt
def isMaven=projectDetails.isMaven
def isLibrary=projectDetails.isLibrary
def isBatch=projectDetails.isBatch
def isWebApp=projectDetails.isWebApp
def isArchetype=projectDetails.isArchetype
def javaVersion = projectDetails.javaVersion
// get JDK Version and javac Version from javaVersion in Config Mgt DB
def jenkinsJdkVersion = jdkMap.get(javaVersion)
def projectJavacVersion = javacMap.get(javaVersion)
println "-> Properties:"
println "-> javaVersion=${javaVersion}"
println "-> projectJavacVersion=${projectJavacVersion}"
println "-> jenkinsJdkVersion=${jenkinsJdkVersion}"
println "-> isMaven=${isMaven}"
println "-> isAnt=${isAnt}"
println "-> webContextRoot=${webContextRoot}"
println "-> mailRecipients=${mailRecipients}"
if ( javaVersion == null || projectJavacVersion == null || jenkinsJdkVersion == null ) {
println "-> ERROR: Should never get here."
return
}
// get additional information from GitLab repository
def gitProjectDetails = getGitProjectDetails(projectId)
def validProjectGruppe = gitProjectDetails.validProjectGruppe
def isGitProjectActive = gitProjectDetails.isActive
def defaultBranch = gitProjectDetails.defaultBranch
println "-> isGitProjectActive=${isGitProjectActive}"
println "-> validProjectGruppe=${validProjectGruppe}"
println "-> defaultBranch=${defaultBranch}"
if( ! isGitProjectActive || defaultBranch == null ) {
println "Skipping project ${gitProjectName} since it is not active or does not have a default branch"
return
}
// each project is assigned a department, therefore all jobs are prefixed with the department id
println "-> Create Job names:"
def ciJobName = "${validProjectGruppe}-${gitProjectName}-1-ci"
def deployJobName = "${validProjectGruppe}-${gitProjectName}-2-deployment"
def robotJobName = "${validProjectGruppe}-${gitProjectName}-3-robot"
// create CI jobs
if(isLibrary && isMaven) {
println "-> create Maven Library Job: ${ciJobName}"
createCIJobOnly(ciJobName, projectGitSshUrlToRepo, defaultBranch, mailRecipients, jenkinsJdkVersion)
} else if(isBatch && isMaven) {
println "-> create Maven Batch Job: ${ciJobName}"
createCIJobOnly(ciJobName, projectGitSshUrlToRepo, defaultBranch, mailRecipients, jenkinsJdkVersion)
} else if(isBatch && isAnt) {
println "-> create Ant Batch Job: ${ciJobName}"
createAntCIJobOnly(ciJobName, gitProjectName, defaultBranch, projectGitSshUrlToRepo, webContextRoot, mailRecipients, jenkinsJdkVersion, projectJavacVersion)
} else if(isMaven && isWebApp ) {
println "-> create Maven WebApp Job: ${ciJobName}"
createMavenCIJob(ciJobName, projectGitSshUrlToRepo, defaultBranch, deployJobName, mailRecipients, jenkinsJdkVersion)
} else if (isAnt && isWebApp) {
println "-> create Ant WebApp Job: ${ciJobName}"
createAntCIJob(ciJobName, gitProjectName, defaultBranch, projectGitSshUrlToRepo, webContextRoot, deployJobName, mailRecipients, jenkinsJdkVersion, projectJavacVersion)
} else if ( isArchetype ) {
createCIJobOnly(ciJobName, projectGitSshUrlToRepo, defaultBranch, mailRecipients, jenkinsJdkVersion)
} else {
println "No CI Jobs will be generated for ${gitProjectName}"
}
// create deployment and acceptance test jobs
if( isWebApp ) {
println "-> create Deploy Job: ${deployJobName}"
createDeployJob(deployJobName, projectGitSshUrlToRepo, defaultBranch, robotJobName, gitProjectName, mailRecipients)
println "-> create Robot Job: ${robotJobName}"
createRobotJob(projectGitSshUrlToRepo, robotJobName, gitProjectName, mailRecipients)
// add CI Job to list
ciJobList += ciJobName
}
}
}
/*
* CREATE VIEWS TO ORGANIZE JOBS
*/
// regular expression prefix for all company departments
def groupPrefix="("
departments.each {
groupPrefix += it+"|"
}
groupPrefix+="NOGROUP)"
createListView('1-Build', 'All Build Jobs', "${groupPrefix}.*-1-ci")
createListView('2-Deploy', 'All Deploy Jobs', "${groupPrefix}.*-2-deployment")
createListView('3-Robot', 'All Robot Jobs', "${groupPrefix}.*-3-robot")
createListView('WebApp', 'All project Jobs', "${groupPrefix}-.*-webapp")
createListView('Batch', 'All Subproject Jobs', "${groupPrefix}-.*-batch-.*")
createListView('Library', 'All Library Jobs', "${groupPrefix}-library-.*")
createListView('Admin', 'All Admin Jobs', 'Administration.*')
createNestedDepartmentView('Department')
nestedView('Build Pipelines') {
description('Automatically generated Build Pipelines for all CI Jobs')
columns {
weather()
}
views {
ciJobList.each {
def job = it
println "Create Build Pipeline View for ${job}"
view(job, type: BuildPipelineView) {
selectedJob(job)
triggerOnlyLatestJob(true)
alwaysAllowManualTrigger(true)
showPipelineParameters(true)
showPipelineParametersInHeaders(true)
showPipelineDefinitionHeader(true)
startsWithParameters(true)
displayedBuilds(5)
}
}
}
}
/*
* JOB DSL UTILITY METHODS
*/
// generate Ant -1-ci Job
def createAntCIJobOnly(def ciJobName, def gitProjectName, def defaultBranch, def projectGitSshUrlToRepo, def webContextRoot, def mailRecipients, def projectJenkinsJdkVersion, def projectJavacVersion) {
job(ciJobName) {
logRotator {
daysToKeep(-1)
numToKeep(10)
}
parameters {
stringParam("project.name", gitProjectName)
stringParam("project.version", defaultBranch)
stringParam("web.context.root", webContextRoot)
booleanParam("junit.skip.tests", false)
stringParam("javac.version", projectJavacVersion)
}
label("ci-slave")
jdk(projectjenkinsJdkVersion)
scm {
git {
remote {
url(projectGitSshUrlToRepo)
credentials(gitCredentials)
}
wipeOutWorkspace(true)
createTag(false)
branch('\${project.version}')
}
}
triggers {
cron("H * * * 1-5")
}
wrappers {
preBuildCleanup()
}
steps {
ant("build") {
buildFile "build.xml"
antInstallation antVersion
javaOpt("-Xmx1G -XX:MaxPermSize=512M")
}
}
publishers {
chucknorris()
archiveJunit("results/junit/**/*.xml")
mailer(mailRecipients, true, true)
}
}
}
// generate Ant -1-ci Job for WebApps
def createAntCIJob(def ciJobName, def gitProjectName, def defaultBranch, def projectGitSshUrlToRepo, def webContextRoot, def deployJobName, def mailRecipients, def projectjenkinsJdkVersion, def projectJavacVersion) {
job(ciJobName) {
logRotator {
daysToKeep(-1)
numToKeep(10)
}
parameters {
stringParam("project.name", gitProjectName)
stringParam("project.version", defaultBranch)
stringParam("web.context.root", webContextRoot)
booleanParam("junit.skip.tests", false)
stringParam("javac.version", projectJavacVersion)
}
label("ci-slave")
jdk(projectjenkinsJdkVersion)
scm {
git {
remote {
url(projectGitSshUrlToRepo)
credentials(gitCredentials)
}
wipeOutWorkspace(true)
createTag(false)
branch('\${project.version}')
}
}
triggers {
cron("H * * * 1-5")
}
wrappers {
preBuildCleanup()
}
steps {
ant("build") {
buildFile "build.xml"
antInstallation antVersion
javaOpt("-Xmx1G -XX:MaxPermSize=512M")
}
}
publishers {
chucknorris()
archiveJunit("results/junit/**/*.xml")
downstreamParameterized {
trigger(deployJobName, 'UNSTABLE_OR_BETTER') {
currentBuild()
}
}
mailer(mailRecipients, true, true)
}
}
}
// generate Maven -1-ci Job for Library or Batch projects
def createCIJobOnly(def ciJobName, def projectGitSshUrlToRepo, def defaultBranch, def mailRecipients, def projectjenkinsJdkVersion) {
job(ciJobName) {
logRotator {
daysToKeep(-1)
numToKeep(10)
}
parameters {
stringParam("project.version", defaultBranch, "Select branch to build: master, branch, tag")
}
label("ci-slave")
jdk(projectjenkinsJdkVersion)
scm {
git {
remote {
url(projectGitSshUrlToRepo)
credentials(gitCredentials)
}
wipeOutWorkspace(true)
createTag(false)
branch('\${project.version}')
}
}
triggers {
scm('H/5 * * * *')
}
wrappers {
preBuildCleanup()
}
steps {
maven {
goals('clean versions:set -DnewVersion=DEV-\${project.version}-\${BUILD_NUMBER} -P jenkins-build -U')
mavenOpts('-Xms256m')
mavenOpts('-Xmx512m')
mavenInstallation(mavenVersion)
}
maven {
goals('org.jacoco:jacoco-maven-plugin:prepare-agent deploy -P jenkins-build,sonar -Dmaven.test.failure.ignore=true')
mavenOpts('-XX:PermSize=256m')
mavenOpts('-XX:MaxPermSize=1024m')
mavenInstallation(mavenVersion)
}
maven {
goals('sonar:sonar -P sonar,jenkins-build')
mavenOpts('-Xmx2G')
mavenOpts('-XX:MaxPermSize=1G')
mavenInstallation(mavenVersion)
}
}
publishers {
chucknorris()
archiveJunit('**/target/surefire-reports/*.xml')
mailer(mailRecipients, true, true)
}
}
}
// generate Maven -1-ci Job for WebApps
def createMavenCIJob(def ciJobName, def projectGitSshUrlToRepo, def defaultBranch, def deployJobName, def mailRecipients, def projectjenkinsJdkVersion) {
job(ciJobName) {
logRotator {
daysToKeep(-1)
numToKeep(10)
}
parameters {
stringParam("project.version", defaultBranch, "Select branch to build: master, branch, tag")
}
label("ci-slave")
jdk(projectjenkinsJdkVersion)
scm {
git {
remote {
url(projectGitSshUrlToRepo)
credentials(gitCredentials)
}
wipeOutWorkspace(true)
createTag(false)
branch('\${project.version}')
}
}
triggers {
scm('H/5 * * * *')
}
wrappers {
preBuildCleanup()
}
steps {
maven {
goals('clean versions:set -DnewVersion=DEV-\${project.version}-\${BUILD_NUMBER} -P jenkins-build -U')
mavenOpts('-Xms256m')
mavenOpts('-Xmx512m')
mavenInstallation(mavenVersion)
}
maven {
goals('org.jacoco:jacoco-maven-plugin:prepare-agent deploy -P jenkins-build,sonar -Dmaven.test.failure.ignore=true')
mavenOpts('-XX:PermSize=256m')
mavenOpts('-XX:MaxPermSize=1024m')
mavenInstallation(mavenVersion)
}
maven {
goals('sonar:sonar -P sonar,jenkins-build')
mavenOpts('-Xmx2G')
mavenOpts('-XX:MaxPermSize=1G')
mavenInstallation(mavenVersion)
}
}
publishers {
chucknorris()
archiveJunit('**/target/surefire-reports/*.xml')
downstreamParameterized {
trigger(deployJobName, 'UNSTABLE_OR_BETTER') {
currentBuild()
}
}
mailer(mailRecipients, true, true)
}
}
}
// generate Deployment -2-deploy Job
def createDeployJob(def deployJobName, def projectGitSshUrlToRepo, def defaultBranch, def robotJobName, def gitProjectName, def mailRecipients ) {
job(deployJobName) {
logRotator {
daysToKeep(-1)
numToKeep(10)
}
label("ci-slave")
multiscm {
git {
remote {
url('git@git:infrastructure/deployment-scripts.git')
credentials(gitCredentials)
}
createTag(false)
branch("master")
}
git {
remote {
url(projectGitSshUrlToRepo)
credentials(gitCredentials)
}
createTag(false)
branch(defaultBranch)
relativeTargetDir(gitProjectName)
}
}
wrappers {
preBuildCleanup()
}
steps {
shell("set -x && sh startDeployment.sh ${gitProjectName}")
}
publishers {
chucknorris()
downstreamParameterized {
trigger(robotJobName, 'UNSTABLE_OR_BETTER') {
currentBuild()
}
}
mailer(mailRecipients, true, true)
}
}
}
// generate Robot -3-robot Job
def createRobotJob(def projectGitSshUrlToRepo, def robotJobName, def gitProjectName, def mailRecipients) {
job(robotJobName) {
logRotator {
daysToKeep(-1)
numToKeep(10)
}
label("ci-robot")
multiscm {
git {
remote {
url(projectGitSshUrlToRepo)
credentials(gitCredentials)
}
createTag(false)
branch("master")
relativeTargetDir(gitProjectName)
}
git {
remote {
url('git@git:test/robot-test-framework.git')
credentials(gitCredentials)
}
createTag(false)
branch("master")
relativeTargetDir("robot-test-framework")
}
}
wrappers {
preBuildCleanup()
}
steps {
ant() {
buildFile "/src/robot/robot.xml"
antInstallation antVersion
}
}
publishers {
chucknorris()
publishRobotFrameworkReports {
passThreshold(100.0)
unstableThreshold(75.0)
onlyCritical(false)
outputPath("/src/robot/log")
reportFileName('report.html')
logFileName('log.html')
outputFileName('output.xml')
disableArchiveOutput(false)
otherFiles('*.jpg', '*.png')
}
mailer(mailRecipients, true, true)
}
}
}
// create list view
def createListView(def jobViewName, def jobDescription, def regularExpression) {
println "createListView ${jobViewName} with ${jobDescription} and ${regularExpression}"
listView(jobViewName) {
description(jobDescription)
filterBuildQueue()
filterExecutors()
jobs {
regex(regularExpression)
}
columns {
status()
buildButton()
weather()
name()
lastSuccess()
lastFailure()
lastDuration()
}
}
}
// create nested view
def createNestedDepartmentView(def viewName) {
println "createNestedDepartmentView for ${viewName}"
nestedView(viewName) {
description('Automatically generated department groups')
columns {
weather()
}
views {
departments.each {
def department = it
println "Create build pipeline subview for ${department}"
view("${department}", type: ListView) {
description("All Jobs for department ${department}")
filterBuildQueue()
filterExecutors()
jobs {
regex("${department}-.*")
}
columns {
status()
buildButton()
weather()
name()
lastSuccess()
lastFailure()
lastDuration()
}
}
}
view("NOGROUP", type: ListView) {
description("All Jobs that are not assigned to a department")
filterBuildQueue()
filterExecutors()
jobs {
regex("NOGROUP-.*")
}
columns {
status()
buildButton()
weather()
name()
lastSuccess()
lastFailure()
lastDuration()
}
}
}
}
}
/////////////////////////////////////////////////////////
// UTILITY METHODS
/////////////////////////////////////////////////////////
// Get project details from Configuration Management Database
// using JDBC and the Git Project Name.
def getProjectDetailsFromConfigMgtDb(def gitProjectName) {
def gitProjectNameQuery = gitProjectName.toUpperCase()
def dbSchema = "configdb"
def dbServer = "dbserver"
def dbUser = 'dbuser'
def dbPassword = 'dbpassword'
// Important: Oracle Driver needs to be available in the classpath
def dbDriver = 'oracle.jdbc.driver.OracleDriver'
def dbUrl = 'jdbc:oracle:thin:@' + dbServer + ':1521:' + dbSchema
sql = Sql.newInstance( dbUrl, dbUser, dbPassword, dbDriver )
// search project in database
def result = sql.firstRow("SELECT * FROM projects WHERE UPPER(project_id) LIKE ${gitProjectNameQuery}")
def isJavaProject=false
def projectDetails = new LinkedHashMap();
if ( result != null ) {
projectDetails.isJavaProject=true
projectDetails.webContextRoot=result.webContextRoot
projectDetails.mailRecipients=result.mailRecipients
projectDetails.isAnt=result.isAnt
projectDetails.isMaven=result.isMaven
projectDetails.isLibrary=result.isLibrary
projectDetails.isBatch=result.isBatch
projectDetails.isWebApp=result.isWebApp
projectDetails.isArchetype=result.isArchetype
projectDetails.javaVersion = result.javaVersion
}
projectDetails
}
// Reads Git Project details via GitLab REST API using Git projectId
def getGitProjectDetails(def projectId) {
def projectDetailsApi = new URL("${gitUrl}/projects/${projectId}?private_token=${gitPrivateToken}")
def projectDetails = new groovy.json.JsonSlurper().parse(projectDetailsApi.newReader())
def projectTags = projectDetails.tag_list
def defaultBranch = projectDetails.default_branch
// determines the Git projects assigned department
// department name is stored in Git Repository Tag.
// defaults to 'NOGROUP'
def validProjectGroup = "NOGROUP"
departments.each {
if ( projectTags != null && projectTags.contains(it) ) {
validProjectGroup = it
}
}
// checks whether the git project is still active or already archived
def isActive = true
if( projectDetails.archived ) {
isActive = false
}
println "-> Get project details from ${gitUrl}/projects/${projectId}?private_token=${gitPrivateToken}"
println "-> validProjectGroup=${validProjectGroup}"
println "-> defaultBranch=${defaultBranch}"
println "-> isActive=${isActive}"
def details = new LinkedHashMap();
details.validProjectGruppe=validProjectGroup
details.defaultBranch=defaultBranch
details.isActive=isActive
details
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment