cauldron_cdh.groovy

// Copyright (c) 2016 Cloudera, Inc. All rights reserved.

// Cauldron CDH6.x Build job on all OSes.
// The OS list is defined in the cdh-local.mk Makefile
// in cdh.git, and is RH6, RH7, Debian8, SLES12, Ubuntu1604
// as of last writing.
package cauldron

import com.cloudera.dsl.Common

def supportedBranches = Common.activeReleases("cdh6")

enum Purpose {
  FULL,
  GERRIT, // the cauldron build run for most components
  GERRIT_CDH, // the cauldron build run for the CDH/cdh repo (needs to be special-cased)
  GERRIT_PATCH, // the cauldron build for patch branches
  ONE_OS,
  MAVEN_ONLY,
  PARAMETERIZED
}

enum Machine {
  ONDEMAND,
  SPOT,
  SPOTBLOCK

  public String label() {
    return toString().toLowerCase()
  }
}

enum MachineSize {
  C5D_9XLARGE,
  C5D_18XLARGE

  // This should translate the value to an AWS machine size
  // Keep in mind that this is only used in the jenkins label,
  // for a new machine size to be used, you should make sure the
  // jenkins label is working first!
  public String label() {
    return toString().toLowerCase().replaceAll('_', '-')
  }
}

/* List of branches that will generate official jobs with no triggers.
 * Useful when we want to keep the job along with its metadata but we
 * only need builds on-demand.
 */
def manualTriggerOnlyBranches = ["6.1.0"]


supportedBranches.each() { branch ->
  makeJob(Purpose.FULL, Machine.SPOTBLOCK, branch, MachineSize.C5D_18XLARGE, null, manualTriggerOnlyBranches.contains(branch))
  makeJob(Purpose.GERRIT_CDH, Machine.SPOT, branch, MachineSize.C5D_18XLARGE)

  if (branch == "cdh6.x") {
    // We special case "cdh6.x" in active releases in order to support hbase incremental auto-merge
    // flow and branch. https://docs.google.com/document/d/14hxUEjrUNC3AOhddCmUJbsMJWyhHDkOqrX-zmyvtQh0/edit#heading=h.wf2h5en873q9
    //
    // The new special case argument allows other branches in different projects get triggered if patch
    // is submitted to gerrit that maches.  It is extensible to enable other auto-merging components
    // such as hadoop to just plug in adding or widening the project/branch spec.
    //
    // TODO We can make this a non-special case if we revisit the branch name convention.  Consider if
    // we changed the hbase branch to cdh6.x-branch-2-merged.  We could then make a new project/branch
    // regex "{branch}-.*-merged" for all projects if they used the convention.
    specialBranches = [ [ project: "cdh/hbase", branch: "cdh6-.*-merged"] ]
    makeJob(Purpose.GERRIT, Machine.SPOT, branch, MachineSize.C5D_9XLARGE, specialBranches)
  } else {
    makeJob(Purpose.GERRIT, Machine.SPOT, branch, MachineSize.C5D_9XLARGE)
  }
}

// For cauldron_parametrized it's ok to use spotblocks because jenkins will fall back
// if we are terminated because of spot termination. The block of 6 hours should
// be enough for most large builds.
makeJob(Purpose.PARAMETERIZED, Machine.SPOTBLOCK, null, MachineSize.C5D_9XLARGE)

// changes to patch branches need a cdh-build to run L0s
makeJob(Purpose.GERRIT_PATCH, Machine.ONDEMAND, null, MachineSize.C5D_9XLARGE)

// one-os build and maven-only are done only for cdh6.x, but we do create a
// full build for all of our supportedBranches.
makeJob(Purpose.MAVEN_ONLY, Machine.SPOT, supportedBranches[0], MachineSize.C5D_9XLARGE )
makeJob(Purpose.ONE_OS, Machine.SPOT, supportedBranches[0], MachineSize.C5D_9XLARGE)

// Adding CDH5 cauldron-based jobs
makeJob(Purpose.FULL, Machine.SPOTBLOCK, "cdh5_cauldron", MachineSize.C5D_18XLARGE)
makeJob(Purpose.MAVEN_ONLY, Machine.SPOT, "cdh5_cauldron", MachineSize.C5D_18XLARGE)

def makeJob(Purpose purpose, Machine machine, String branch, MachineSize machineSize, specialcase = [], Boolean manualTriggerOnly = false) {
  def jobName = "cauldron"
  if (branch) {
    jobName += "_" + branch;
  }
  if (purpose != Purpose.FULL) {
    jobName += "_" + purpose.toString().toLowerCase()
  }

  freeStyleJob(jobName) {
    description =  "Cauldron patch build job (${purpose == Purpose.GERRIT_PATCH ? "patch" : "branch"})"
    if (machine == Machine.SPOT) {
      description += "\n<b>Build running on EC2 spot instances; beware machine disappearance!"
    }
    Common.description(delegate, "cauldron-dev@cloudera.com", "cauldron-dev", description)
    Common.defaults(delegate)

    // We set the defaults here, then we modify the variables as needed below
    // Branch name need to be specified even if we modify it later,
    // otherwise it could end up being blank and the job rescheduling infinitely
    githubBranch = purpose == Purpose.GERRIT_PATCH ? "cdh6.x" : branch
    numLogsToKeep = 100
    noteRegex = '^(GBN .*)'
    entrypoint = "tools/jenkins.sh"
    // Only user-driven (gerrit and parameterized) jobs are concurrent
    isConcurrentBuild = false
    hasTriggers = true
    scriptOverrides = ""
    postBuildScript = ""
    allowEmptyArtifacts = false
    githubRepo = "CDH/cdh"
    maxConcurrentBuilds = -1

    switch (purpose) {
      case Purpose.FULL:
        entrypoint = "tools/jenkins.sh official"
        break
      case Purpose.GERRIT:
      case Purpose.GERRIT_CDH:
      case Purpose.GERRIT_PATCH:
        entrypoint = "tools/gerrit-build.sh"
        isConcurrentBuild = true
        allowEmptyArtifacts = true
        maxConcurrentBuilds = 10
        break
      case Purpose.ONE_OS:
        scriptOverrides = "export OS=redhat7 ACTIONS='dangerous-clean clean bootstrap-download sync-shallow package'"
        break
      case Purpose.MAVEN_ONLY:
        scriptOverrides = "export ACTIONS='dangerous-clean clean bootstrap-download sync-shallow maven-only'"
        break
      case Purpose.PARAMETERIZED:
        githubBranch = '$BRANCH'
        numLogsToKeep = 1000
        isConcurrentBuild = true
        githubRepo = '$REPO'
        parameters {
          stringParam("OS", "", "Comma-separated list of platforms to build for: \
                    redhat7,redhat6,ubuntu1604,sles12,debian8. Leave this blank to build all platforms")
          stringParam("REPO", "CDH/cdh", "The CDH repository to be used in this build")
          stringParam("BRANCH", "cdh6.x", "Branch to be used in this build")
          stringParam("NOTE", "", "A short note about what this build or your ID. This will be used to set description on jenkins build.")
          stringParam("ACTIONS", "dangerous-clean clean bootstrap-download sync-shallow package publish-s3", "Build actions e.g. dangerous-clean, clean, sync, maven-only, package, publish-s3. Space separated.")
          textParam("CDH_JSON_OVERRIDE", "", '''\
                    Partial cdh.json specification. The build will checkout all repos as defined in cdh.json, but remotes and branches can be
                    overriden by providing a partial specification.<br>
                    <b> Example: </b><br>
                    <pre>
                    {
                      "remotes": { "custom": {"fetch": "git://github.infra.cloudera.com/<b>USERNAME</b>/%(name)s.git" }},
                      "projects": {
                        "<b>COMPONENT</b>": { "track-branch": "<b>BRANCH</b>", "remotes": ["custom"] }
                      }
                    }
                    </pre>
                    '''.stripIndent())
          booleanParam("RUN_BVTS", false, "After, a succesful build, trigger the Ad-hoc-CDH-BVT job with the current GBN")
          booleanParam("RUN_SMOKES", false, "After, a succesful build, trigger the Ad-hoc-CDH-Smokes job with the current GBN")
          labelParam('WORKER_LABEL') {
            size = machineSize.label()
            type = machine.label()
            defaultValue("cauldron-docker-ubuntu1604-ec2-${size}-${type}")
            description('Worker label on which to run the job.')
          }
        }
        break
    }

    logRotator {
      numToKeep(numLogsToKeep)
    }

    if (purpose != Purpose.GERRIT_CDH) {
      Common.internalRepoBasedGithub(delegate, githubRepo, githubBranch)
    } else {
      // the cdh/cdh repo changes need to be built by checking out gerrit refspecs directly
      scm {
        git {
          remote {
            url ("https://gerrit.sjc.cloudera.com/cdh/cdh")
            // GERRIT_REFSPEC is provided by the Gerrit Plugin.
            // TODO: docs say this is not set in the "ref updated case", and
            // suggests using $GERRIT_REFNAME instead.  Need to figure out if
            // that is needed or not.
            refspec('$GERRIT_REFSPEC')
          }
          extensions {
            cloneOptions { timeout(30) }
            localBranch('$GERRIT_BRANCH')
              scmName {
                name('gerrit')
              }
            choosingStrategy {
              gerritTrigger()
            }
          }
        }
      }
    }

    triggers {
      if (EnumSet.of(Purpose.GERRIT, Purpose.GERRIT_CDH, Purpose.GERRIT_PATCH).contains(purpose)) {
        gerrit {
          configure {
            // The job runs on silent mode since we do our own reporting
            // in gerrit-build.py, this is needed since the jenkins plugin
            // doesn't provide a way to review patchsets other than the one
            // that triggered the job.
            // Note that silentMode(true) here is not enough, the configure
            // block that gets passed here is the raw GerritTrigger xml node.
            def silentMode = it / 'silentMode'
            silentMode.setValue(true)

            // Drafts should not trigger anything.
            def excludeDrafts = it / 'triggerOnEvents' / 'com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginPatchsetCreatedEvent' / 'excludeDrafts'
            excludeDrafts.setValue(true)
          }
          events {
            patchsetCreated()
            draftPublished()
          }
          if (purpose == Purpose.GERRIT_PATCH) {
            project('reg_exp:(?!Impala-auxiliary-tests|cdh/(?:cdh|sqoark)).*', "reg_exp:cdh6.*_patch\\d+")
          } else if (purpose == Purpose.GERRIT) {
            // use negative lookahead assertions to match everything except cdh/cdh, Impala-auxiliary-tests and cdh/sqoark.
            // Once sqoark is added to the platform cauldron build, we'll remove it from here so it is built as well.
            project('reg_exp:(?!Impala-auxiliary-tests|cdh/(?:cdh|sqoark)).*', branch)
            for (s in specialcase) {
              project(s.project, s.branch)
            }
          } else if (purpose == Purpose.GERRIT_CDH) {
            // match only the cdh/cdh project
            project('cdh/cdh', branch)
          }
        }
      } else if (purpose != Purpose.PARAMETERIZED) {
        // Only set up cron triggers  when not on playground and the manual trigger flag is off
        if (!Common.onPlayground() && !manualTriggerOnly) {
          githubPush()
          // Weekday at 11 AM and 11 PM
          cron('H 11,23 * * 1-5')
        }
      }
    }

    if (purpose != Purpose.PARAMETERIZED) {
      size = machineSize.label()
      type = machine.label()
      label("cauldron-docker-ubuntu1604-ec2-${size}-${type}")
    }

    // Since this is a user-driven job, we allow concurrent builds.
    concurrentBuild(isConcurrentBuild)
    if (maxConcurrentBuilds > 0) {
      // Concurrent builds, throttling to maxConcurrentBuilds total
      throttleConcurrentBuilds {
        maxPerNode(1)
        maxTotal(maxConcurrentBuilds)
      }
    }
    steps {
      if (purpose == Purpose.PARAMETERIZED) {
        // Set build description ahead of time, so it's easy to find your build
        // amongst running ones.
        buildDescription(null, "\$BUILD_USER_ID \$NOTE <br> \$REPO \$BRANCH")
      }
      if (purpose == Purpose.GERRIT_PATCH) {
        // For Gerrit patches, cdh branch can be calculated only when triggered
        // gerrit_branch typically looks like cdh6.1.0_patch1234_patch4321
        shell '''\
          #!/usr/bin/env bash
          git checkout ${GERRIT_BRANCH%%_*}'''.stripIndent()
      }

      shell """\
        #!/usr/bin/env bash
        ${scriptOverrides}
        ${entrypoint}""".stripIndent()
      // Temporary hook to wait for L0s to finish and post back result.
      // Mostly because we don't want to poll in the expensive VM.
      // Remove once we have a separate service to handle the testing and postback.
      // See: QAINFRA-4483
      if (purpose == Purpose.GERRIT || purpose == Purpose.GERRIT_CDH) {
        conditionalSteps {
          condition {
            fileExists('quanta/precommit-l0-postback.properties', BaseDir.WORKSPACE)
          }
          steps {
            downstreamParameterized {
              trigger('precommit-l0-postback') {
                parameters {
                  currentBuild()
                  propertiesFile('quanta/precommit-l0-postback.properties')
                }
              }
            }
          }
        }
      }
    }

    publishers {
      archiveArtifacts {
        // Gerrit jobs can 'skip' the build.
        allowEmpty(allowEmptyArtifacts)
        pattern('local-output/build.json, local-output/manifest.html, local-output/manifest.json, local-output/gbn.txt, local-output/quanta_token.txt, local-output/logs/*')
      }
      descriptionSetterPublisher {
        regexp(noteRegex)
        regexpForFailed(noteRegex)
        description('')
        descriptionForFailed('')
        setForMatrix(false)
      }
      if (!EnumSet.of(Purpose.GERRIT, Purpose.GERRIT_CDH, Purpose.GERRIT_PATCH).contains(purpose)) {
        extendedEmail {
          disabled(Common.onPlayground())
          contentType('text/html')
          body = '''\
            <p><a href="${BUILD_URL}">${PROJECT_NAME}#${BUILD_NUMBER}</a> - $BUILD_STATUS</p>
            <h2>Build Log (last 500 lines):</h2>
            <pre>
            ${BUILD_LOG,maxLines=500,escapeHtml=true}
            </pre>'''.stripIndent()
          triggers {
            if (purpose == Purpose.PARAMETERIZED) {
              success { sendTo { requester() } }
              failure {
                subject('Failed: $PROJECT_DEFAULT_SUBJECT')
                sendTo { requester() }
                content(body)
              }
            } else {
              failure {
                recipientList("cauldron-dev@cloudera.com")
                subject('Failed: $PROJECT_DEFAULT_SUBJECT')
                content(body)
              }
            }
          }
        }
      }
    }

    wrappers {
      if (purpose == Purpose.FULL) {
        credentialsBinding {
          file('CAULDRON_GPG_SIGNING_KEY_FILE', 'GPG_SIGNING_KEY_FILE')
          file('CAULDRON_GPG_SIGNING_PASSPHRASE_FILE', 'GPG_SIGNING_PASSPHRASE_FILE')
          file('CAULDRON_ARTIFACTORY_PASSPHRASE_FILE', 'ARTIFACTORY_SNAPSHOTS_FILE')
          file('CAULDRON_PYPIRC_FOR_PYPI_INFRA_CLOUDERA_COM', 'PYPIRC_FOR_PYPI_INFRA_CLOUDERA')
        }
      }
      colorizeOutput()
      timeout {
        noActivity(7200)
        failBuild()
        writeDescription('Build failed due to timeout after {0} minutes')
      }

      // Need to set buildUserVars so that we can pass user info to downstream jobs
      //  triggered via jenkinsCLI
      buildUserVars()
    }
  }
}