Doc publishing steps in Jenkins

DocSteps.groovy

package steps

import validation.PipelineParametersWrapper

class DocsSteps extends BaseSteps {
    public static final String NIGHTLY_DOCS_PATH = 'nightlies.testing.ansible.com:/var/www/html/docs_7d5d0dae81129579'
    public static final String RELEASE_DOCS_PATH = 'releases-master.ansible.com:/var/www/html/docs'
    public static final String ANSIBLE_NIGHTLY_DOCS_PATH = "${NIGHTLY_DOCS_PATH}/ansible"
    public static final String ANSIBLE_RELEASE_DOCS_PATH = "${RELEASE_DOCS_PATH}/ansible"

    DocsSteps(Object script, PipelineParametersWrapper paramsWrapper) {
        super(script, paramsWrapper)
    }

    def checkoutAnsible(Map stepParameters=[:]) {
        script.checkout([
                $class: 'GitSCM',
                branches: [[name: "*/${stepParameters.ansibleBranch ?: paramsWrapper.docsParameters.ansibleBranch}" ]],
                extensions: [
                        [
                                $class: 'SubmoduleOption',
                                disableSubmodules: false,
                                parentCredentials: false,
                                recursiveSubmodules: true,
                                reference: '',
                                trackingSubmodules: false
                        ]
                ],
                userRemoteConfigs: [
                        [
                                url: "https://github.com/${stepParameters.ansibleFork ?: paramsWrapper.docsParameters.ansibleFork}/ansible.git"
                        ]
                ]
        ])
    }

    def checkoutProductsDocsRelativePath(Map stepParameters = [:]) {
        String branch = stepParameters.productDocsBranch ?: paramsWrapper.docsParameters.productDocsBranch
        String fork = stepParameters.productDocsFork ?: paramsWrapper.docsParameters.productDocsFork

        script.checkout([
                $class           : 'GitSCM',
                branches         : [[name: "${branch}"]
                ],
                extensions       : [
                        [
                                $class   : 'CloneOption',
                                noTags   : false,
                                reference: '',
                                shallow  : true
                        ],
                        [
                                $class           : 'RelativeTargetDirectory',
                                relativeTargetDir: 'product-docs'
                        ]
                ],
                userRemoteConfigs: [
                        [
                                credentialsId: 'foo',
                                url          : "git@github.com:${fork}/product-docs.git"
                        ]
                ]
        ])
    }

    def checkoutAnsibleDocsite(Map stepParameters=[:]) {
        script.checkout([
                $class: 'GitSCM',
                branches: [[name: "*/${stepParameters.ansibleBranch ?: paramsWrapper.docsParameters.ansibleBranch}" ]],
                extensions: [
                        [
                                $class: 'SubmoduleOption',
                                disableSubmodules: false,
                                parentCredentials: false,
                                recursiveSubmodules: true,
                                reference: '',
                                trackingSubmodules: false
                        ]
                ],
                userRemoteConfigs: [
                        [
                                credentialsId: 'foo',
                                url: "git@github.com:${stepParameters.ansibleFork ?: paramsWrapper.docsParameters.ansibleFork}/docsite.git"
                        ]
                ]
        ])
    }

    def installRequirements() {
        script.sh """
            python3 -m pip install --upgrade pip
            python3 -m pip install setuptools six
            
            if [ -f "requirements.txt" ] ; then
                # Importing ansible as part of the build so we need ansible's runtime deps
                python3 -m pip install -r requirements.txt
            fi
            
            pushd docs/docsite

            if [ -f "known_good_reqs.txt" ] ; then
                # on branches where this file exists, install known good versions of the docs build reqs
                python3 -m pip install -r known_good_reqs.txt
            elif [ -f "../../test/sanity/code-smell/docs-build.requirements.txt" ] ; then
                # installing good requirements from sanity tests
                python3 -m pip install -r ../../test/sanity/code-smell/docs-build.requirements.txt
            else 
                # otherwise, install generic docs build requirements
                python3 -m pip install -r requirements.txt
            fi
            
            popd
        """
    }

    def setupPythonSymlink(Map stepParameters = [:]) {
        String target = stepParameters.targetPython ?: paramsWrapper.docsParameters.targetPython
        script.sh """
            mkdir -p /home/jenkins/agent/.local/bin/
            ln -s -f ${target} /home/jenkins/agent/.local/bin/python && python --version
            ln -s -f ${target} /home/jenkins/agent/.local/bin/python3 && python3 --version
        """
    }

    def buildDocs(Map stepParameters = [:]) {
        String packageVersion = stepParameters.packageVersion ?: paramsWrapper.docsParameters.packageVersion
        String language = stepParameters.language ?: paramsWrapper.docsParameters.language
        Boolean includeBreadcrumbs = stepParameters.language ?: paramsWrapper.docsParameters.includeBreadcrumbs

        // We need to properly provide CPUS for openshift environment.
        // See: https://major.io/2019/04/05/inspecting-openshift-cgroups-from-inside-the-pod/
        String cpus = stepParameters.cpus ?: 1
        this.script.withEnv([
                "CPUS=${cpus}"
        ]) {
            script.sh """
                pushd docs/docsite
                PACKAGE_VERSION=${packageVersion}
                LANGUAGE=${language}
                INCLUDE_BREADCRUMBS="${PipelineParametersWrapper.toYesNo(includeBreadcrumbs)}"
    
                VERSION="\${PACKAGE_VERSION}"
    
                # Set collection list - if version is devel, set to null
                if [ "\$PACKAGE_VERSION" == "devel" ]; then
                    COLLECTION_LIST=""
                  else
                    COLLECTION_LIST="\${PACKAGE_VERSION}"
                fi
                
                # Add language suffix to URL for translated docs
                if [ "\${LANGUAGE}" == "japanese" ]; then
                  # Use #_ja in URL for translated docs
                  VERSION="\${VERSION}_ja"
                fi
                
                # Turn off collection-level breadcrumbs if jenkins build is having memory failures
                if [ "\${INCLUDE_BREADCRUMBS}" == "no" ]; then
                  # create .antsibull.cfg with breadcrumbs = false to turn them off
                  echo "breadcrumbs = false" > ~/.antsibull.cfg
                fi
                
                make webdocs ANSIBLE_VERSION="\${COLLECTION_LIST}"
                
                if [ "\${INCLUDE_BREADCRUMBS}" == "no" ]; then
                  # remove config file when done
                  rm ~/.antsibull.cfg
                fi
                popd
            """
        }
    }

    def publishDocs(Map stepParameters = [:]) {
        String rsyncTarget = stepParameters.rsyncTarget ?: ANSIBLE_NIGHTLY_DOCS_PATH
        String packageVersion = stepParameters.packageVersion ?: paramsWrapper.docsParameters.packageVersion
        String language = stepParameters.language ?: paramsWrapper.docsParameters.language
        Boolean addLatestSymlink = stepParameters.addLatestSymlink ?: paramsWrapper.docsParameters.addLatestSymlink

        script.sshagent(credentials : ['foo']) {
            script.sh """
                pushd docs/docsite
                PACKAGE_VERSION=${packageVersion}
                LANGUAGE=${language}
                ADD_LATEST_SYMLINK=${PipelineParametersWrapper.toYesNo(addLatestSymlink)}

                RSYNC_OPTS="-a --delete --progress --exclude=.repo.lock --exclude .git/\\*"
                RSYNC_RSH="ssh -o User=jenkins"
                
                # By default we publish to docs.testing.ansible.com dir
                RSYNC_TARGET="${rsyncTarget}"
                
                VERSION="\${PACKAGE_VERSION}"
                
                # Add language suffix to URL for translated docs
                if [ "\${LANGUAGE}" == "japanese" ]; then
                  # Use #_ja in URL for translated docs
                  VERSION="\${VERSION}_ja"
                fi    
                
                rsync \${RSYNC_OPTS} --rsh "\${RSYNC_RSH}" _build/html/ "\${RSYNC_TARGET}/\$VERSION/"
                
                # add 'latest' as a symlink for this build/version
                if [ "\${ADD_LATEST_SYMLINK}" == "yes" ]; then
                  ln -s \$VERSION _build/latest
                  rsync \${RSYNC_OPTS} --rsh "\${RSYNC_RSH}" _build/latest "\${RSYNC_TARGET}"
                fi
                popd
            """
        }
    }

    def publishDevelopment(Map stepParameters = [:]) {
        String gitBranch = stepParameters.ansibleBranch ?: paramsWrapper.docsParameters.ansibleBranch

        this.publishDocs(stepParameters + [
                rsyncTarget: ANSIBLE_NIGHTLY_DOCS_PATH
        ])

        if (gitBranch == 'origin/devel') {
            this.syncReleaseMirrors()
        }
    }

    def publishRelease(Map stepParameters = [:]) {
        this.publishDocs(stepParameters + [
                rsyncTarget: ANSIBLE_RELEASE_DOCS_PATH
        ])

        syncReleaseMirrors()
    }

    def buildAndPublishDocsiteDev(Map stepParameters = [:]) {
        this.buildAndPublishDocsite(stepParameters + [
                rsyncTarget: NIGHTLY_DOCS_PATH
        ])
    }

    def buildAndPublishDocsiteProd(Map stepParameters = [:]) {
        this.buildAndPublishDocsite(stepParameters + [
                rsyncTarget: RELEASE_DOCS_PATH
        ])
    }

    private buildAndPublishDocsite(Map stepParameters = [:]) {
        String rsyncTarget = stepParameters.rsyncTarget
        assert rsyncTarget
        script.sshagent(credentials : ['foo']) {
            script.sh """
                #!/bin/bash -xe
    
                RSYNC_OPTS="-a --progress"
                RSYNC_RSH="ssh -o User=jenkins"
                RSYNC_TARGET="${rsyncTarget}"
                
                # the rsync command must include all files and directories that should go on the server
                # including .htaccess files for redirects, the robots.txt, and the sitemaps
                #
                rsync \${RSYNC_OPTS} --rsh "\${RSYNC_RSH}" *.html .htaccess *sitemap.xml robots.txt ansible assets "\${RSYNC_TARGET}"
            """
        }

    }

    // TODO: Move sync job to a new master
    private def syncReleaseMirrors() {
        script.sh """
            # Sync release mirrors
            echo -e "build" > sync_releases.cfg
        """
    }

    String getDocsBranchFromAapVersion(String aapVersion) {
        String towerVersion = script.platformVersion.getControllerVersionInfo(aapVersion)['version']
        if (towerVersion == 'devel') {
            return 'main'
        } else if (towerVersion == '4.0.0') {
            return "release_4.0"
        } else {
            return "release_${towerVersion}"
        }
    }
}