Jenkinsfile.groovy

#!/usr/bin/env groovy
/**
 * common Pipeline
 *
 * used global variable:
 * STAGE_PORT: ssh port to deployer host
 * STAGE_USER: ssh user to deployer host
 * STAGE_SRV: host name for deploy
 * STAGE_SRV_WRK: host name for deploy worker
 *
 * Used variables:
 * DOCKER_DIR: directory containing the docker-compose yml files
 * ImageName: name of the image do deploy
 * gitName: git repo name
 * branch: branch to deploy, also use as tag for the docker image
 * DOCKER_REGISTRY: docker registry host
 * dockerRevision: docker revision build as {branch}-{build-id}
 * deployToEnv: choose the deploy destination, if == 'none' do not deploy
*/

def NO_MULTI_BRANCH_PROJECT = [
  'Internal/api-creator',
  'Internal/api-docs'
]
if (NO_MULTI_BRANCH_PROJECT.contains(env.JOB_NAME)) {
  echo 'NOT A MULTI BRANCH PROJECT'
  doJob()
}

def doJob() {

  def DONT_DEPLOY_PROJECTS = [
    'api-all',
    'api-skeleton',
    'wrk-grapher'
  ]
  def GO_PROJECTS = [
    'api-creator',
    'api-image-importer',
    'api-integration'
  ]
  def ANGULAR_PROJECTS = [
    'Dashboard-angular',
    'Onekloud-cockpit'
  ]
  def PYTHON_PROJECTS = [
    'Discovery-python'
  ]
  def NODE_PROJECTS = [
    'wrk-grapher',
    'wrk-newsletter',
    'common-node'
  ]

  //def DOCKER_REGISTRY='docker.onekloud.net'
  def COMMIT_ID = env.GIT_COMMIT
  def PULL_REQUEST_ID = env.CHANGE_ID
  def BUILD_NODE = null
  def DOCKER_DIR = 'api'
  def BRANCH_TO_DEPLOY = [
    'develop',
    'master'
  ]

  def dockerImageName = null
  def dockerRevision = null

  def projectVersion = null

  def folderMultiBranch = null
  def projectName = null
  def typeBranch = null
  def gitBranch = null

  def jobInfos = env.JOB_NAME.split('/')
  if (jobInfos.length == 3) {
    folderMultiBranch = jobInfos[0]
    projectName = jobInfos[1]
    typeBranch = jobInfos[2]
    gitBranch = jobInfos[2]
  } else if (jobInfos.length == 2) {
    folderMultiBranch = jobInfos[0]
    projectName = jobInfos[1]
    typeBranch = 'master'
    gitBranch = 'master'
  }

  if (isMultiBranch(folderMultiBranch, projectName)) {
    // check git flow branching only if multi branch
    def branchInfos = env.BRANCH_NAME.split('/')
    if (branchInfos.length == 2) {
      typeBranch = branchInfos[0]
      gitBranch = env.BRANCH_NAME
    }
  }
  def workspacePath = getWorkspace(projectName, gitBranch, GO_PROJECTS)


  echo """
  ___________________________________________________________
  | env.branch   |   ${env.BRANCH_NAME}
  | folderMulti  |   ${folderMultiBranch}
  | projectName  |   ${projectName}
  | typeBranch   |   ${typeBranch}
  | gitBranch    |   ${gitBranch}
  | dockerDir    |   ${DOCKER_DIR}
  | dockerImage  |   ${dockerImageName}
  ___________________________________________________________
  """

  // by default deploy build images
  def deployToEnv = "none"
  // by default deploy build images
  def enableDockerBuild = true
  def enableDockerPush = true

  // Check if specific node (to build) is required
  if (isSlave7Specific(projectName)) {
    BUILD_NODE = 'Slave7-New-medium'
  }

  // define env to deploy to and docker build
  // no Docker image, nor deployment for common Java lib
  if (projectName.startsWith('common-')) {
    enableDockerBuild = false
    enableDockerPush = false
  } else if (typeBranch == 'develop') {
    deployToEnv = 'stage';
  } else if (typeBranch == 'release') {
    deployToEnv = 'pre-prod';
  } else if (typeBranch == 'master') {
    if (folderMultiBranch == 'Fronts' || projectName == 'wrk-grapher') { // don't rebuild the image and reuse the build from pre-prod
      enableDockerBuild = false
      enableDockerPush = false
    }
    deployToEnv = 'prod';
  } else {
    enableDockerBuild = false
    enableDockerPush = false
  }

  // Don't deploy
  if (DONT_DEPLOY_PROJECTS.contains(projectName)) {
    deployToEnv = 'none';
  }

  // set docker image name
  // force lowercase name -> following docker standard
  dockerImageName = projectName.toLowerCase()
  // define docker dir

  if (projectName.startsWith('wrk-') || projectName == 'Discovery-python') {
    DOCKER_DIR = 'worker'
  } else if (projectName.startsWith('api-')) {
    DOCKER_DIR = 'api'
  } else if (projectName == 'api-image-importer') {
    DOCKER_DIR = 'api-image-importer'
  } else if (projectName == 'Dashboard-angular') {
    if (typeBranch == 'release') {
      DOCKER_DIR = 'dashboard-testing'
    } else {
      DOCKER_DIR = 'dashboard'
    }
  // } else if (projectName == 'Dashboard-login') {        DEPRECATED
  //   DOCKER_DIR = 'dashboard-login'                      DEPRECATED
  } else if (projectName == 'Onekloud-cockpit') {
    DOCKER_DIR = 'onekloud-cockpit'
  }


  node (BUILD_NODE) {
    ansiColor('xterm') {
      try {
        notifyBuild('STARTED')
        ws(workspacePath) {
          // checkout stage is common for all prokects
          stage('Checkout') {
            echo """
              Using:
              checkingout: git@bitbucket.org:onekloud/${projectName}.git
              gitName: ${projectName}
              ImageName: ${DOCKER_REGISTRY}/${dockerImageName}:${gitBranch}
              branch: ${gitBranch}
              BUILD:
              deployToEnv: ${deployToEnv}
              enableDockerBuild:${enableDockerBuild}
              ImageName: ${dockerImageName}
              DOCKER_DIR: ${DOCKER_DIR}
              Docker_Login: ${DOCKER_USERNAME}
              Docker_Pass: ${DOCKER_PASSWORD}
              PULL_REQUEST_ID: ${PULL_REQUEST_ID}
              """
            if (DOCKER_DIR.contains('worker')) {
                STAGE_SERVER = env.STAGE_WRK
            }else {
                STAGE_SERVER = env.STAGE_SRV
            }

            if (env.BRANCH_NAME)
              checkout scm
            else
              checkout([
                $class: 'GitSCM',
                branches: [[name: "*/${gitBranch}"]],
                doGenerateSubmoduleConfigurations: false,
                extensions: [],
                submoduleCfg: [],
                userRemoteConfigs: [[credentialsId: '5e7c76cf-72a2-4488-81da-77dd40488bd5',
                url: "git@bitbucket.org:onekloud/${projectName}.git"]]
              ])
          }

          // Angular projects
          if (ANGULAR_PROJECTS.contains(projectName)) {

            stage('Yarn') {sh """yarn"""}

            stage('Lint') {sh """npm run lint"""}

            // stage('Tests') {sh """npm run test-ci"""}

            stage('Sonar') {
              // requires SonarQube Scanner 2.8+
              def scannerHome = tool 'sonarq';
              sh '''
                echo $(node -p -e "require(\'./package.json\').version") > versionFile
              '''
              projectVersion = readFile 'versionFile'
              projectVersion = projectVersion.trim()
              def branchSonar = (gitBranch == 'master' || typeBranch == 'release') ? 'master' : 'develop'
              withSonarQubeEnv('Sonarq') {
                // can add
                sh """${scannerHome}/bin/sonar-scanner \
                    -Dsonar.projectKey=${projectName} \
                    -Dsonar.projectName=${projectName} \
                    -Dsonar.projectVersion=${projectVersion} \
                    -Dsonar.branch=${branchSonar} \
                    -Dsonar.sourceEncoding=UTF-8 \
                    -Dsonar.sources=src/app \
                    -Dsonar.exclusions=**/node_modules/**,**/*.spec.ts \
                    -Dsonar.tests=src/app \
                    -Dsonar.test.inclusions=**/*.spec.ts \
                    -Dsonar.ts.tslint.configPath=tslint.json \
                    -Dsonar.javascript.lcov.reportPath=./coverage/lcov.info"""
              }
            }

            stage('Build') {

              if (deployToEnv == 'none') {
                deployToEnv = 'dev'
              }
              if (deployToEnv == 'pre-prod') {
                deployToEnv = 'prod'
              }
              sh """#!/bin/bash -l
                echo "<!-- autoBuild: # ${BUILD_ID} | ${COMMIT_ID} | ${projectVersion} | `date -R` -->" >> src/index.html
                echo "npm run build:${deployToEnv}"
                set -o pipefail;  npm run build:${deployToEnv} | grep -v 'building modules' | grep -v 'Dropping un' | grep -v 'Side effects in initialization of unused variable'
              """
              if (deployToEnv == 'dev') {
                deployToEnv = 'none'
              }
              if (typeBranch == 'release') {
                deployToEnv = 'pre-prod';
              }
            }
          }

          // GO-lang projects
          else if (GO_PROJECTS.contains(projectName)) {

            stage('Install GO') {
              env.GOPATH = "/home/jenkins/workspace"
              env.PATH = "${env.GOPATH}/bin:${env.PATH}"
              try {
                sh """#!/bin/bash -l
                  git config --global url."git@bitbucket.org:".insteadOf "https://bitbucket.org/"
                  git config --global url."git@github.com:".insteadOf "https://gihthub.com/"
                  """

              } catch (error) {
                  echo "Error on getting dep"
              }
            }

            stage('Get Dependencies') {
              sh """#!/bin/bash -l
                govend -v
              """
            }

            stage('Tests') {
              sh """#!/bin/bash -l
                go test
              """
            }

            stage('Go build') {
              sh """#!/bin/bash -l
                go build -v -i -o ${projectName}
              """
            }

          }

          else if (PYTHON_PROJECTS.contains(projectName)) {

            stage('Sonar') {
              // requires SonarQube Scanner 2.8+
              def scannerHome = tool 'sonarq';
              withSonarQubeEnv('Sonarq') {
                sh """${scannerHome}/bin/sonar-scanner \
                    -Dsonar.projectKey=${projectName} \
                    -Dsonar.projectName=${projectName} \
                    -Dsonar.projectVersion=0.1 \
                    -Dsonar.sources=app \
                    -Dsonar.sourceEncoding=UTF-8"""
              }
            }

          }

          else if (NODE_PROJECTS.contains(projectName)) {

            stage('Yarn') {sh """yarn"""}

            stage('Lint') {sh """npm run lint"""}

            def testStatus = sh(returnStdout: true, script: 'npm test').trim()
            if (testStatus.contains("Error: no test specified")) {
              currentBuild.result='UNSTABLE'
            } else {
              stage('Tests & Code Coverage') {sh """npm run coverage"""}
            }

            stage('Sonar') {
              def srcFolders = ""
              if (projectName.startsWith('common-')) {
                srcFolders = "src,index.js"
              } else if (projectName.startsWith('common-')) {
                srcFolders = "src,app.js"
              } else {
                srcFolders = "src"
              }
              // requires SonarQube Scanner 2.8+
              def scannerHome = tool 'sonarq';
              sh '''
                echo $(node -p -e "require(\'./package.json\').version") > versionFile
              '''
              projectVersion = readFile 'versionFile'
              projectVersion = projectVersion.trim()
              def branchSonar = (gitBranch == 'master' || typeBranch == 'release') ? 'master' : 'develop'
              withSonarQubeEnv('Sonarq') {
                // can add
                sh """${scannerHome}/bin/sonar-scanner \
                    -Dsonar.projectKey=${projectName} \
                    -Dsonar.projectName=${projectName} \
                    -Dsonar.projectVersion=${projectVersion} \
                    -Dsonar.branch=${branchSonar} \
                    -Dsonar.sourceEncoding=UTF-8 \
                    -Dsonar.sources=${srcFolders} \
                    -Dsonar.exclusions=node_modules/**/*.*,coverage/**/*.*,test/**/*.*  \
                    -Dsonar.language=js \
                    -Dsonar.tests=test \
                    -Dsonar.javascript.lcov.reportPaths=coverage/lcov.info"""
              }
            }

          }

          // Gradle: common Project
          else {

            if (typeBranch == 'master' || typeBranch == 'develop') {
              stage('Sonar') {
                
              }
            }

            stage('Build') {
              if (projectName.startsWith('common-')) {
                sh """gradle build --refresh-dependencies install upload"""
              } else {
                  sh """gradle build --refresh-dependencies install shadowJar"""
                try {
                  sh """gradle upload"""
                } catch (error) {
                  echo "Error on uploading archive"
                }
              }
            }

          }

          if (enableDockerBuild) {

            // define tag for docker image
            if ((folderMultiBranch == 'Fronts' || projectName == 'wrk-grapher') && typeBranch == 'release') {
              dockerRevision = "${projectVersion}"
            } else {
              dockerRevision = "${gitBranch}-${env.BUILD_ID}"
            }

            stage('Build Docker image') {
              echo "Deploy version ${dockerRevision} job:${gitBranch}"
              if ((folderMultiBranch == 'Fronts' || projectName == 'wrk-grapher') && typeBranch == 'release') {
                sh """#!/bin/bash -l
                  set -o pipefail; docker build --no-cache -t ${DOCKER_REGISTRY}/${dockerImageName}:${projectVersion} . | grep -v 'Sending build context to Docker daemon'
                """
              } else {
                sh """#!/bin/bash -l
                  set -o pipefail; docker build --no-cache -t ${DOCKER_REGISTRY}/${dockerImageName}:${gitBranch} -t ${DOCKER_REGISTRY}/${dockerImageName}:${dockerRevision} . | grep -v 'Sending build context to Docker daemon'
                """
              }
            }

          }

          if (enableDockerPush) {

            stage('Push Docker images') {
              sh """#!/bin/bash -l
                docker login -u='${DOCKER_USERNAME}' -p='${DOCKER_PASSWORD}' ${DOCKER_REGISTRY};
                set -o pipefail; docker push ${DOCKER_REGISTRY}/${dockerImageName} | grep -v Preparing | grep -v 'Layer already exists' | grep -v Waiting
                curl -X POST \
                --data-urlencode 'payload={ "channel": "#docker-container", "username": "Whalee", "text": "New container pushed for ${dockerImageName}\nVersion: ${dockerRevision}\nBranch: ${gitBranch}", "icon_url": "https://s3.amazonaws.com/assets-onekloud/onekloud/jenkins/docker.png" }' \
                https://hooks.slack.com/services/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
              """
            }

          }

          //////////////////////
          // deploy docker containes
          // excepted for api-all
          if (deployToEnv == 'stage') {
            stage ('Pull Docker Image') {
              sshagent(['bd919fe3-6211-4dcc-b87f-f67e4de49cf7']) {
                sh "ssh -o \"StrictHostKeyChecking=no\" -p ${STAGE_PORT} ${STAGE_USER}@${STAGE_SERVER} /usr/local/bin/docker-compose -f /srv/docker-compose/${DOCKER_DIR}/docker-compose.yml pull ${dockerImageName}"
              }
            }
            stage ('Kill running container') {
              sshagent(['bd919fe3-6211-4dcc-b87f-f67e4de49cf7']) {
                sh "ssh -o \"StrictHostKeyChecking=no\" -p ${STAGE_PORT} ${STAGE_USER}@${STAGE_SERVER} /usr/local/bin/docker-compose -f /srv/docker-compose/${DOCKER_DIR}/docker-compose.yml kill ${dockerImageName}"
              }
            }
            stage ('Clean old container') {
              sshagent(['bd919fe3-6211-4dcc-b87f-f67e4de49cf7']) {
                sh "ssh -o \"StrictHostKeyChecking=no\" -p ${STAGE_PORT} ${STAGE_USER}@${STAGE_SERVER} /usr/local/bin/docker-compose -f /srv/docker-compose/${DOCKER_DIR}/docker-compose.yml rm -f ${dockerImageName}"
              }
            }
            stage ('Run new container') {
              sshagent(['bd919fe3-6211-4dcc-b87f-f67e4de49cf7']) {
                sh "ssh -o \"StrictHostKeyChecking=no\" -p ${STAGE_PORT} ${STAGE_USER}@${STAGE_SERVER} /usr/local/bin/docker-compose -f /srv/docker-compose/${DOCKER_DIR}/docker-compose.yml up -d ${dockerImageName}"
              }
            }
          }

          // only for dashboard for now
          if (deployToEnv == 'pre-prod') {
            stage ('Pull Docker Image') {
              sshagent(['bd919fe3-6211-4dcc-b87f-f67e4de49cf7']) {
                sh "ssh -o \"StrictHostKeyChecking=no\" -p ${STAGE_PORT} ${STAGE_USER}@${STAGE_SRV} 'export projectVersion=${projectVersion} && /usr/local/bin/docker-compose -f /srv/docker-compose/${DOCKER_DIR}/docker-compose.yml pull'"
              }
            }
            stage ('Kill running container') {
              sshagent(['bd919fe3-6211-4dcc-b87f-f67e4de49cf7']) {
                sh "ssh -o \"StrictHostKeyChecking=no\" -p ${STAGE_PORT} ${STAGE_USER}@${STAGE_SRV} 'export projectVersion=${projectVersion} && /usr/local/bin/docker-compose -f /srv/docker-compose/${DOCKER_DIR}/docker-compose.yml kill'"
              }
            }
            stage ('Clean old container') {
              sshagent(['bd919fe3-6211-4dcc-b87f-f67e4de49cf7']) {
                sh "ssh -o \"StrictHostKeyChecking=no\" -p ${STAGE_PORT} ${STAGE_USER}@${STAGE_SRV} 'export projectVersion=${projectVersion} && /usr/local/bin/docker-compose -f /srv/docker-compose/${DOCKER_DIR}/docker-compose.yml rm -f'"
              }
            }
            stage ('Run new container') {
              sshagent(['bd919fe3-6211-4dcc-b87f-f67e4de49cf7']) {
                sh "ssh -o \"StrictHostKeyChecking=no\" -p ${STAGE_PORT} ${STAGE_USER}@${STAGE_SRV} 'export projectVersion=${projectVersion} && /usr/local/bin/docker-compose -f /srv/docker-compose/${DOCKER_DIR}/docker-compose.yml up -d'"
              }
            }
            // only notify the first time the release branch is build
            if (projectName == 'Dashboard-angular' && env.BUILD_NUMBER == '1') {
              sh """#!/bin/bash -l
                curl -X POST -H 'Content-type: application/json' \
                --data '{ "channel": "#general", "username": "Unicorn", "text": "Heads Up ! :eyes:\nv${projectVersion} of the dashboard will be online soon. :tada:\nCheck it out before it is too late !", "attachments": [{"color": "#7CD197", "title_link": "http://app.testing.onekloud.net/", "title": "Pre Production Website | v${projectVersion}", "fallback": "A new dashboard version is in the pipe. Check it out !"}], "icon_url": "https://s3.amazonaws.com/assets-onekloud/onekloud/jenkins/unicorn.png" }' \
                https://hooks.slack.com/services/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
                curl -X POST -H 'Content-type: application/json' \
                --data '{ "channel": "#engineering", "username": "Unicorn", "text": "Heads Up ! :eyes:\nv${projectVersion} of the dashboard will be online soon. :tada:\nCheck it out before it is too late !", "attachments": [{"color": "#7CD197", "title_link": "http://app.testing.onekloud.net/", "title": "Pre Production Website | v${projectVersion}", "fallback": "A new dashboard version is in the pipe. Check it out !"}], "icon_url": "https://s3.amazonaws.com/assets-onekloud/onekloud/jenkins/unicorn.png" }' \
                https://hooks.slack.com/services/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
                curl -X POST -H 'Content-type: application/json' \
                --data '{ "channel": "#frontend", "username": "Unicorn", "text": "Heads Up ! :eyes:\nv${projectVersion} of the dashboard will be online soon. :tada:\nCheck it out before it is too late !", "attachments": [{"color": "#7CD197", "title_link": "http://app.testing.onekloud.net/", "title": "Pre Production Website | v${projectVersion}", "fallback": "A new dashboard version is in the pipe. Check it out !"}], "icon_url": "https://s3.amazonaws.com/assets-onekloud/onekloud/jenkins/unicorn.png" }' \
                https://hooks.slack.com/services/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
              """
            }
          }

          if (deployToEnv == 'prod') {
            stage('deploy prod') {
              echo "Deploy PROD todo"
            }
          }
        // END java
        } // end WS dir
      } catch (e) {
        // If there was an exception thrown, the build failed
        currentBuild.result = "FAILED"
        throw e
      } finally {
        // Success or failure, always send notifications
        notifyBuild(currentBuild.result)
      }
    }
  }
}

def isMultiBranch(String folderMultiBranch, String projectName) {
  def isMulti = false;
  def MULTI_BRANCH_PROJECTS = [
    'Fronts',
    'Multi-Apis',
    'Multi-Workers'
  ]
  if (MULTI_BRANCH_PROJECTS.contains(folderMultiBranch) || (folderMultiBranch == 'Internal' && projectName == 'api-skeleton')) {
    isMulti = true
  }
  return isMulti
}

def isSlave7Specific(String projectName) {
  def slave7Build = false;
  def PROJECTS_SLAVE_7 = [
    'Dashboard-angular',
    // 'Dashboard-login',         DEPRECATED
    'Onekloud-cockpit',
    'api-image-importer',
    'api-integration',
    'api-creator',
    'wrk-grapher',
    'wrk-newsletter',
    'common-node'
  ]
  if (PROJECTS_SLAVE_7.contains(projectName)) {
    slave7Build = true
  }
  return slave7Build
}

def getWorkspace(String projectName, String gitBranch, List<String> GO_PROJECTS) {
  def workspace = "workspace/${projectName}/${gitBranch}".replace('%2F', '_');
  if (GO_PROJECTS.contains(projectName)) {
    workspace = "workspace/src/bitbucket.org/onekloud/${projectName}".replace('%2F', '_');
  }
  return workspace
}

def notifyBuild(String buildStatus = 'STARTED') {
  // build status of null means successful
  buildStatus = buildStatus ?: 'SUCCESS'

  // Default values
  def colorName = 'RED'
  def colorCode = '#FF0000'
  def subject = "${buildStatus}: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'"
  def summary = "${subject} (${env.BUILD_URL})"
  def details = """<p>STARTED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]':</p>
    <p>Check console output at &QUOT;<a href='${env.BUILD_URL}'>${env.JOB_NAME} [${env.BUILD_NUMBER}]</a>&QUOT;</p>"""
  def frontDevChannels = ['@ulysse']

  // Override default values based on build status
  if (buildStatus == 'STARTED') {
    color = 'BLUE'
    colorCode = '#3083c6'
  } else if (buildStatus == 'SUCCESS') {
    color = 'GREEN'
    colorCode = '#00FF00'
  } else if (buildStatus == 'UNSTABLE') {
    color = 'YELLOW'
    colorCode = '#FFFA00'
  } else {
    color = 'RED'
    colorCode = '#FF0000'
  }

  // Send notifications to slack
  slackSend (color: colorCode, message: summary)
  // send notification in private for front only
  def frontJob = [
    'Fronts/Dashboard-angular/master',
    'Fronts/Dashboard-angular/develop',
    // 'Fronts/Dashboard-login/master',         DEPRECATED
    // 'Fronts/Dashboard-login/develop',        DEPRECATED
    'Fronts/Onekloud-cockpit/master',
    'Fronts/Onekloud-cockpit/develop'
  ]
  if (frontJob.contains(env.JOB_NAME)) {
    for (dev in frontDevChannels) {
      slackSend (channel: dev, color: colorCode, message: summary)
    }
  }
}
return this