Script to migrate old Build/Project errors into new `Notification` objects

migrate_notifications.py

# Build objects to create Notification for
#
#
# In [10]: Build.objects.filter(date__year__in=[2023, 2024]).exclude(error="").count()
# Out[10]: 1231365


# Different types of Notification messages we need
#
#
# In [3]: Build.objects.filter(date__year__in=[2023, 2024]).values_list("error", flat=True).order_by("error").distinct("error").count()
# Out[3]: 28859


# Query with regex
#
#
# In [36]: regex = 'Problem in your project\'s configuration. Invalid "(.*)": expected one of \((.*)\), got (.*)'
# In [37]: list(Build.objects.filter(date__year__in=[2023, 2024], error__regex=regex).values_list("error", flat=True)[:10])
# In [38]: re.match(regex, build.error).groups()
# Out[38]: ('python.version', '2, 2.7, 3, 3.5, 3.6, 3.7, 3.8', '3.11')


import re
import structlog

from readthedocs.doc_builder.exceptions import *
from readthedocs.projects.exceptions import *
from readthedocs.config.exceptions import *

from readthedocs.projects.notifications import MESSAGE_PROJECT_SKIP_BUILDS


log = structlog.get_logger(__name__)


simple_messages_mapping = {
    'A configuration file was not found. Make sure you have a conf.py file in your repository.': ProjectConfigurationError.NOT_FOUND,
    'A configuration file was not found. Make sure you have a "mkdocs.yml" file in your repository.': MkDocsYAMLParseError.NOT_FOUND,
    'Build cancelled by user.': BuildCancelled.CANCELLED_BY_USER,
    'Build exited due to excessive memory consumption': BuildUserError.BUILD_EXCESSIVE_MEMORY,
    'Build exited due to time out': BuildUserError.BUILD_TIME_OUT,
     'No "_readthedocs/html" folder was created during this build.': BuildUserError.BUILD_COMMANDS_WITHOUT_OUTPUT,
    'No TeX files were found': BuildUserError.TEX_FILE_NOT_FOUND,
    'No TeX files were found.': BuildUserError.TEX_FILE_NOT_FOUND,
    'PDF file was not generated/found in "_readthedocs/pdf" output directory.': BuildUserError.BUILD_OUTPUT_HAS_NO_PDF_FILES,
    'PDF generation failed. The build log below contains information on what errors caused the failure.Our code has recently changed to fail the entire build on PDF errors, where we used to pass the build when a PDF was created.Please contact us if you need help understanding this error.': BuildUserError.GENERIC,
    'Please make sure the MkDocs YAML configuration file is not empty.': MkDocsYAMLParseError.EMPTY_CONFIG,
    "Some files were detected in an unsupported output path, '_build/html'. Ensure your project is configured to use the output path '$READTHEDOCS_OUTPUT/html' instead.": BuildUserError.BUILD_OUTPUT_OLD_DIRECTORY_USED,
    'Symlinks are not fully supported': UnsupportedSymlinkFileError.SYMLINK_USED,
    'The configuration file required to build documentation is missing from your project. Add a configuration file to your project to make it build successfully. Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html': BuildUserError.NO_CONFIG_FILE_DEPRECATED,
    'The configuration key "build.image" is deprecated. Use "build.os" instead to continue building your project. Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html#build-os': BuildUserError.BUILD_IMAGE_CONFIG_KEY_DEPRECATED,
    'The configuration key "build.os" is required to build your documentation. Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html#build-os': BuildUserError.BUILD_OS_REQUIRED,
    'The "docs_dir" config from your MkDocs YAML config file does not contain a valid path.': MkDocsYAMLParseError.INVALID_DOCS_DIR_CONFIG,
    'We encountered a problem with a command while building your project. To resolve this error, double check your project configuration and installed dependencies are correct and have not changed recently.': BuildUserError.GENERIC,
    "We found more than one conf.py and are not sure which one to use. Please, specify the correct file under the Advanced settings tab in the project's Admin.": ProjectConfigurationError.MULTIPLE_CONF_FILES,
    "You can not have two versions with the name latest or stable. Ensure you don't have both a branch and a tag with this name.": RepositoryError.DUPLICATED_RESERVED_VERSIONS,
    "Your documentation did not generate an 'index.html' at its root directory. This is required for documentation serving at the root URL for this version.": BuildUserError.BUILD_OUTPUT_HTML_NO_INDEX_FILE,
    'There was a problem cloning your repository. Please check the command output for more information.': RepositoryError.GENERIC,
    'There was a problem connecting to your repository, ensure that your repository URL is correct and your repository is public. Private repositories are not supported.': RepositoryError.CLONE_ERROR_WITH_PRIVATE_REPO_ALLOWED,

    'This build was manually skipped using a command exit code.': BuildUserError.SKIPPED_EXIT_CODE_183,
}


regex_messages_mapping = {
    'Build output directory for format "htmlzip" does not contain any files. It seems the build process created the directory but did not save any file to it.': {
        "message_id": BuildUserError.BUILD_OUTPUT_HAS_0_FILES,
        "format_values": {
            "artifact_type": "htmlzip",
        },
    },
    'Build output directory for format "pdf" contains multiple files and it is not currently supported. Please, remove all the files but the "pdf" your want to upload.': {
        "message_id": BuildUserError.BUILD_OUTPUT_HAS_MULTIPLE_FILES,
        "format_values": {
            "artifact_type": "pdf",
        },
    },
    'Build output directory for format "pdf" contains multiple files and it is not currently supported. Please, remove all the files but the "pdf" you want to upload.': {
        "message_id": BuildUserError.BUILD_OUTPUT_HAS_MULTIPLE_FILES,
        "format_values": {
            "artifact_type": "pdf",
        },
    },
    'Build output directory for format "pdf" does not contain any files. It seems the build process created the directory but did not save any file to it.': {
        "message_id": BuildUserError.BUILD_OUTPUT_HAS_0_FILES,
        "format_values": {
            "artifact_type": "pdf",
        },
    },
    r'Failed to checkout revision: (.+)': {
        "message_id": RepositoryError.FAILED_TO_CHECKOUT,
        "format_values": {
            # Index position from `re.match().groups()` list
            "revision": 0,
        },
    },
    # "One or more submodule URLs are not valid: ['android/app/src/main/java/in/beaconcha/mobile/widget'], git/ssh URL schemas for submodules are not supported.": "",
    r"Problem in your project's configuration. Configuration file not found in: (.+)": {
        "message_id": ConfigError.CONFIG_PATH_NOT_FOUND,
        "format_values": {
            "directory": 0,
        },
    },
    r'Problem in your project\'s configuration. Invalid "(.+)": ([\.a-z]+): Invalid configuration option: .+. Make sure the key name is correct.': {
        "message_id": ConfigError.GENERIC_INVALID_CONFIG_KEY,
        "format_values": {
            "key": 0,
            "source_file": 1,
            "error_message": "",
        },
    },
 r'Problem in your project\'s configuration. Invalid "build.apt_packages[0]": expected string': {
        "message_id": ConfigValidationError.INVALID_STRING,
        "format_values": {
            "key": "build.apt_packages[0]",
            "value": "unknown",
        },
 },
 r'Problem in your project\'s configuration. Invalid "build.commands": expected list': {
        "message_id": ConfigValidationError.INVALID_LIST,
        "format_values": {
            "key": "build.commands",
            "value": "unknown",
        },
 },
    r'Problem in your project\'s configuration. Invalid "(.*)": expected one of \((.*)\), got (.*)': {
        "message_id": ConfigValidationError.INVALID_CHOICE,
        "format_values": {
            "key": 0,
            "choices": 1,
            "value": 2,
        },
    },
    r'The "(.*)" config from your MkDocs YAML config file has to be a list of relative paths.': {
        "message_id": MkDocsYAMLParseError.INVALID_EXTRA_CONFIG,
        "format_values": {
            "config": 0,
        },
    },
    r"The file (.*) doesn't exist. Make sure it's a valid file path.": {
        "message_id": UserFileNotFound.FILE_NOT_FOUND,
        "format_values": {
            "filename": 0,
        },
    },
    r'There was a problem with Read the Docs while building your documentation. Please try again later. If this problem persists, report this error to us with your build id \((\d+)\).': {
        "message_id": BuildAppError.GENERIC_WITH_BUILD_ID,
        # We don't need format values here since it uses the Build instance from where it's attached to
        "format_values": {},
    },
 r'This build was terminated due to inactivity. If you continue to encounter this error, file a support request with and reference this build id \((\d+)\).': {
        "message_id": BuildAppError.BUILD_TERMINATED_DUE_INACTIVITY,
        # We don't need format values here since it uses the Build instance from where it's attached to
        "format_values": {},
 },
    r'Your mkdocs.yml could not be loaded, possibly due to a syntax error(.*)': {
        "message_id": MkDocsYAMLParseError.SYNTAX_ERROR,
        # We don't need format values here since it uses the Build instance from where it's attached to
        "format_values": {},
    }
}

# Only migrate this and last year builds to the new notifications system
# We can expand this if we want, but it will require more message mappings.
queryset = Build.objects.filter(date__year__in=(2023, 2024))


# Migrate simple errors that have a complete match
for error, message_id in simple_messages_mapping.items():
    count = queryset.filter(error=error).count()
    log.info("Builds to migrate.", count=count, error=error)
    for build in queryset.filter(error=error):
        log.info("Creating notification.", message_id=message_id)
        Notification.objects.add(
            attached_to=build,
            message_id=message_id,
            dismissable=False,
        )


# Migrate errors that require a regex and/or need format values
for pattern, data in regex_messages_mapping.items():
    count = queryset.filter(error__regex=pattern).count()
    log.info("Builds to migrate.", count=count, pattern=pattern)
    for build in queryset.filter(error__regex=pattern):
        message_id = data.get("message_id")
        groups = re.match(pattern, build.error).groups()
        if groups:
            format_values = {}
            for key, value in data.get("format_values").items():
                if isinstance(value, int):
                    format_values[key] = groups[value]
                else:
                    format_values[key] = value
        else:
            format_values = data.get("format_values")

        log.info("Creating notification.", error=build.error, groups=groups, message_id=message_id, format_values=format_values)
        Notification.objects.add(
            attached_to=build,
            message_id=message_id,
            dismissable=False,
            format_values=format_values,
        )


# Migrate notifications for projects skipped
for project in Project.objects.filter(skip=True):
    log.info("Creating project skipped notification.", project_slug=project.slug)
    Notification.objects.add(
        attached_to=project,
        message_id=MESSAGE_PROJECT_SKIP_BUILDS,
        dismissable=False,
    )