Jenkins declarative Pipeline in Kubernetes with Parallel and Sequential steps

00-infra.yaml

apiVersion: v1
kind: Pod
spec:
  # dnsConfig:
  #   options:
  #     - name: ndots
  #       value: "1"
  containers:
  - name: dind
    image: abdennour/docker:19-dind-bash
    command:
    - cat
    tty: true
    volumeMounts:
    - name: dockersock
      readOnly: true
      mountPath: /var/run/docker.sock
    resources:
      limits:
        cpu: 1000m
        memory: 768Mi
  volumes:
  - name: dockersock
    hostPath: 
      path: /var/run/docker.sock

01-sequential.Jenkinsfile

// Sequential

pipeline {
  agent {
    kubernetes {
      defaultContainer 'jnlp'
      yamlFile '00-infra.yaml'
    }
  }
  stages {
    stage('Build') {
      steps {
        container('dind') {
          sh 'docker build --network=host portal/ -t portal:dev'
          sh 'docker build --network=host iam/ -t iam:dev'
          sh 'docker push portal:dev'
          sh 'docker push iam:dev'
        }
        containerLog 'dind'
      }
    }
  }
}

02-parallel.Jenkinsfile

pipeline {
  agent any
  stages {

    stage('Build') {
      parallel {
        // stage 1-a
        stage('build-portal') {
          agent {
            kubernetes {
              defaultContainer 'jnlp'
              yamlFile '00-infra.yaml'
            }
          }
          steps {
            container('dind') {
               sh 'docker build --network=host portal/ -t portal:dev'
               sh 'docker push portal:dev'
            }
            
          }
        }
        // stage 1-b
        stage('build-iam') {
          agent {
            kubernetes {
              defaultContainer 'jnlp'
              yamlFile '00-infra.yaml'
            }
          }
          steps {
            container('dind') {
               sh 'docker build --network=host iam/ -t iam:dev'
               sh 'docker push iam:dev'
            }
          }
        }
      }
    }
  }
}

03-parallel-loop.Jenkinsfile

def services = ['portal', 'iam']


def parallelBuildStagesMap = services.collectEntries {
    ["${it}" : generateBuildStage(it)]
}

def generateBuildStage(service) {
    return {
        stage("build-${service}") {
          agent {
            kubernetes {
              defaultContainer 'jnlp'
              yamlFile '00-infra.yaml'
            }
          }
          steps {
            container('dind') {
              sh 'docker build -t ${service}:dev'
              sh 'docker push ${service}:dev' 
            }
          }
        }
    }
}

pipeline {
  agent any
  stages {

    stage('Build') {
       steps {
        script { // <-- script must be used to integrate imperative pipeline into Declarative
          parallel parallelBuildStagesMap
        }
      }   
    } // end of main stage Build
  } // end of stages
} // end of pipeline

Hello! I've been trying to get 03-parallel-loop.Jenkinsfile to work for a couple hours now and I think I might be running into an update to Jenkins having broken it. When I try to do a similar thing I get the error
No such DSL method 'agent' found among steps
So it looks like we are trying to put a stage within a steps which is not allowed. We cannot remove the steps because the script block needs to be in it.
Not sure how else to fix this.
Any ideas?
My code

def buildStages

// Create List of build stages to suit
def prepareBuildStages() {
  def buildParallelMap = [:]
  for (name in [ 'one', 'two', 'three' ] ) {
    buildParallelMap.put(name, prepareOneBuildStage(name))
  }
  return buildParallelMap
}

def prepareOneBuildStage(String name) {
  return {
    stage("deploy-${name}") {
      agent {
        kubernetes {
          yamlFile 'build-agent.yaml'
          defaultContainer 'terraform'
        }
      }
      steps {
        container(name: "terraform", shell: "/bin/bash") {
          println("Building ${name}")
          sh(script:'sleep 5', returnStatus:true)
        }
      }
    }
  }
}

pipeline {
  agent none
  stages {
    stage('Initialise') {
      agent { label 'master' }
      steps {
        script {
          buildStages = prepareBuildStages()
          println("Initialised pipeline.")
        }
      }
    }
    stage('Deploy') {
      steps {
        script {
          parallel buildStages
        }
      }   
    }
  }
}

So it looks like we are trying to put a stage within a steps which is not allowed. We cannot remove the steps because the script block needs to be in it.

Yeah. I too have the same problem. Also: java.lang.NoSuchMethodError: No such DSL method 'agent' found among steps

@harshavmb You need to use agent directive inside prepareOneBuildStage function, agent is for declarative pipeline only

@sarg3nt @harshavmb

Here's the fix of OP's script

def services = ['portal', 'iam']


def parallelBuildStagesMap = services.collectEntries {
    ["${it}": generateBuildStage(it)]
}

def generateBuildStage(service) {
    return {
        node(POD_LABEL) {
            stage("build-${service}") {
                steps {
                    container('dind') {
                        sh 'docker build -t ${service}:dev'
                        sh 'docker push ${service}:dev'
                    }
                }
            }
        }
    }
}

pipeline {
    kubernetes {
        yamlFile '00-infra.yaml'
    }
    stages {
        stage('Build') {
            steps {
                script { // <-- script must be used to integrate imperative pipeline into Declarative
                    parallel parallelBuildStagesMap
                }
            }
        } // end of main stage Build
    } // end of stages
} // end of pipeline

But this will run both in parallel in one agent?

I'm struggling to find a way how to run code-generated stages with kubernetes plugin agents in parallel but using one agent per stage.

@toabi Does my script fit your needs?

Actually I figured out how to run one agent stage:

This is getDeployStage.groovy:

def call(target, String tagName) {
  return [
      "${target.cluster} ${target.release}@${tagName}",
      {
        agentK8s {
          node(POD_LABEL) {
            stage("${target.cluster}/${target.namespace}/${target.release}/${tagName}") {
               doSomethingInSomeContainer()
            }
          }
        }
      }
  ]
}

While agentK8s is defined in vars/agentK8s.groovy as such. This will create a new agent, and inside the closure the POD_LABEL variable is accessible which is used above in the node(POD_LABEL)

def call(body) {
  podTemplate(
      containers: [
          containerTemplate(
              name: "kubectl",
              image: "bitnami/kubectl:1.22",
              command: "sleep",
              args: "infinity",
              runAsUser: "0"
          ),
          containerTemplate(
              name: "helm",
              image: "alpine/helm:3.8.1",
              command: "sleep",
              args: "infinity",
              runAsUser: "0"
          )
      ]
  ) {
    body.call()
  }
}

Additionally there's something which creates a list of the getDeployStage outputs, for example like that:

deployBranchStages.groovy

def call(deployTargets, currentBranch) {
    return deployTargets.findAll({it.branch == currentBranch}).collectEntries { target ->
        return getDeployStage(target, env.pushedImageTag)
    }
}

So in the end in the pipeline the stage with dynamic parallel stages with one agent per stage looks like that:

stage('Deploy Branch') {
    when {
        beforeAgent true
        expression { opts.deploy.targets.size > 0 }
    }
    steps {
        script {
            parallel deployBranchStages(opts.deploy.targets, BRANCH_NAME)
        }
    }
}