some-scripts.py

milovidov@milovidov-desktop:~/work/sandbox/BaseOnCommitTask$ cat base_text_test_task.py 
# -*- coding: utf-8 -*-
import logging
from sandbox import sdk2
import sandbox.common.types.resource as ctr
import sandbox.common.types.client as ctc

from sandbox.projects.clickhouse.BaseOnCommitTask.base import PostStatuses, NeedToRunDescription
from sandbox.projects.clickhouse.BaseOnCommitTask.test_task import BaseOnCommitTestTask
from sandbox.projects.clickhouse.resources import CLICKHOUSE_REPO_NO_SUBMODULES
from sandbox.projects.clickhouse.util.task_helper import has_changes_in_code, decompress_fast


class BaseTextTestTask(BaseOnCommitTestTask):

    class Requirements(BaseOnCommitTestTask.Requirements):
        cores = 1
        ram = 8192  # 8GiB or less
        client_tags = ctc.Tag.MULTISLOT

        class Caches(sdk2.Requirements.Caches):
            pass  # means that task do not use any shared caches

    @classmethod
    def get_resources(cls, commit, repo, pull_request):
        logging.info("Searching for CLICKHOUSE_REPO_NO_SUBMODULES at commit %s", commit.sha)
        resources = sdk2.Resource.find(
            CLICKHOUSE_REPO_NO_SUBMODULES,
            state=ctr.State.READY,
            attrs=dict(commit=commit.sha, pr_number=pull_request.number)
        ).order(-CLICKHOUSE_REPO_NO_SUBMODULES.id).limit(1)
        logging.info("Search finished")
        return resources.first()

    def post_statuses(self):
        return PostStatuses.ALWAYS

    def _save_repo(self, commit, repo, pull_request):
        logging.info("Downloading resource")
        repo_resource = BaseTextTestTask.get_resources(commit, repo, pull_request)
        repo_data = sdk2.ResourceData(repo_resource)
        repo_path = str(repo_data.path)
        logging.info("Download finished, archive path %s", repo_path)

        logging.info("Unpacking dowloaded repo")
        decompress_fast(repo_path)

        logging.info("Unpack finished")
        return "./ClickHouse"

    def check(self, repo_path, commit, repo, pull_request):
        raise Exception("Unimplemented")

    def run(self, commit, repo, pull_request):
        logging.info("Start fetching resource with repo")
        repo_path = self._save_repo(commit, repo, pull_request)

        try:
            check_lines, output_path = self.check(repo_path, commit, repo, pull_request)
            if not check_lines:
                self.Parameters.description = self.get_context_name() + " passed"
                return "success", self.Parameters.description, [(self.get_context_name(), "OK")], output_path
            else:
                self.Parameters.description = self.get_context_name() + " Erros:\n" + "\n".join([line[0] for line in check_lines])
                return "failure", "Found {} errors".format(len(check_lines)), check_lines, output_path
        except Exception as ex:
            logging.info("Exception happened", ex)
            return "error", "Cannot run " + self.get_context_name(), None, None

milovidov@milovidov-desktop:~/work/sandbox/ClickhouseDocsCheck$ cat __init__.py 
# -*- coding: utf-8 -*-
import logging
import sandbox.sdk2.helpers
import subprocess

import os
from sandbox import sdk2
import sandbox.common.types.resource as ctr
import sandbox.common.types.client as ctc
from sandbox.projects.clickhouse.BaseOnCommitTask.base import NeedToRunDescription
from sandbox.projects.clickhouse.BaseOnCommitTask.base_text_test_task import BaseTextTestTask
from sandbox.projects.clickhouse.resources import CLICKHOUSE_DOCS_LXC_CONTAINER
from sandbox.projects.clickhouse.util.task_helper import has_changes_in_documentation


class ClickhouseDocsCheck(BaseTextTestTask):

    class Requirements(BaseTextTestTask.Requirements):
        privileged = True
        client_tags = ctc.Tag.GENERIC

    class Parameters(BaseTextTestTask.Parameters):
        _container = sdk2.parameters.Container(
            "Environment container resource",
            resource_type=CLICKHOUSE_DOCS_LXC_CONTAINER,
        )

    def on_create(self):
        self.Parameters._container = sdk2.Resource.find(
            CLICKHOUSE_DOCS_LXC_CONTAINER,
            state=ctr.State.READY,
            attrs=dict(released="stable")
        ).order(-CLICKHOUSE_DOCS_LXC_CONTAINER.id).first().id

    @staticmethod
    def need_to_run(gh_helper, pr_info):
        if 'pr-backport' in pr_info.labels or 'release' in pr_info.labels:
            return NeedToRunDescription(False, 'Not ran for backport or release PRs', True)

        if not has_changes_in_documentation(gh_helper, pr_info.pygithub_commit, pr_info.pygithub_pr):
            return NeedToRunDescription(False, "No diff in .md, .html, etc...", True)

        return NeedToRunDescription(True)

    @staticmethod
    def get_context_name():
        return "Docs check"

    def install_newer_python(self):
        with sandbox.sdk2.helpers.ProcessLog(self, logger="insall_python") as pl:
            cmd = "add-apt-repository ppa:deadsnakes/ppa -y && "\
                  " apt-get update && apt-get -o DPkg::Options::=\"--force-confold\" install python3.7 python3.7-venv --yes --force-yes && "\
                  " update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 && python3 -m ensurepip --upgrade"
            retcode = subprocess.Popen(cmd, shell=True, stderr=pl.stdout, stdout=pl.stdout).wait()
            if retcode != 0:
                raise Exception("Cannot prepare new python")

    def check(self, repo_path, commit, repo, pull_request):
        logging.info("Docs check started")
        output_name = "docs_output.txt"
        lines = []
        self.install_newer_python()
        with sandbox.sdk2.helpers.ProcessLog(self, logger="docs_output") as pl:
            retcode = subprocess.Popen(
                "/bin/bash -c \"set -o pipefail && cd {}/docs/tools && python3 -m pip install --ignore-installed --upgrade setuptools pip virtualenv \
                && mkdir venv && virtualenv -p $(which python3) venv && source venv/bin/activate \
                && python3 -m pip install --ignore-installed -r requirements.txt && ./build.py --skip-git-log 2>&1 | tee {}\"".format(repo_path, output_name),
                shell=True, stderr=pl.stdout, stdout=pl.stdout).wait()
            if retcode == 0:
                logging.info("Docs check finished")
            else:
                logging.info("Check failed")
                lines.append(("Return code is not 0", "FAIL"))
                log_path = str(pl.stdout.path)

        output_path = "{}/docs/tools/{}".format(repo_path, output_name)
        if os.path.exists(output_path):
            logging.info("Output exists %s", output_path)
            with open(output_path, 'r') as check_file:
                for line in check_file:
                    if "ERROR" in line:
                        lines.append((line.split(':')[-1], "FAIL"))
        else:
            output_path = log_path
            logging.info("Output %s doesn't exists", output_path)

        return lines, output_path