Drupal 8 common migration tasks

config.yml

# CircleCI integration with Drupal 8.
# Notice: this file goes at .circleci/config.yml but Gists don't allow subdirectories.
version: 2.1

# Reusable steps.

## Defines images and working directory.
defaults: &defaults
  working_directory: /var/www/html

executors:
  docker_drupal8ci:
    docker:
    - image: juampynr/drupal8ci:latest
  docker_drupal8ci_mariadb:
    docker:
    - image: juampynr/drupal8ci:latest
    - image: quay.io/some-org/repository:latest
      auth:
        username: $QUAY_USERNAME_V2
        password: $QUAY_PASSWORD_V2
    - image: memcached:1.5
  docker_drupal8ci_mariadb_selenium:
    docker:
    - image: juampynr/drupal8ci:latest
    - image: selenium/standalone-chrome-debug:3.141.59
    - image: quay.io/some-org/repository:latest
      auth:
        username: $QUAY_USERNAME_V2
        password: $QUAY_PASSWORD_V2
    - image: memcached:1.5

## Defines the cache restoring mechanism.
restore_cache: &restore_cache
  # We use the composer.lock as a way to determine if we can cache our build.
  keys:
    - v4-dependencies-{{ .Branch }}-{{ checksum "composer.lock" }}
    # fallback to using the latest master cache if no exact match is found
    - v4-dependencies-master-

## Defines the cache saving mechanism.
save_cache: &save_cache
  paths:
    - ./vendor
    - /root/.composer/cache
  key: v4-dependencies-{{ .Branch }}-{{ checksum "composer.lock" }}

#Jobs

# Install composer libraries and other code dependencies.
composer_install: &composer_install
  <<: *defaults
  executor: docker_drupal8ci
  steps:
    - checkout
    - restore_cache: *restore_cache
    - run:
        name: Configure environment
        command: robo composer:install
    - persist_to_workspace:
        root: /var/www/html
        paths: .
    - save_cache: *save_cache

## Job to update the source database.
migrate: &migrate
  <<: *defaults
  executor: docker_drupal8ci_mariadb_selenium
  steps:
    - attach_workspace:
        at: /var/www/html
    - setup_remote_docker:
        docker_layer_caching: true
    - run:
        name: Install Docker client
        command: |
          set -x
          VER="18.09.3"
          curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
          tar -xz -C /tmp -f /tmp/docker-$VER.tgz
          mv /tmp/docker/* /usr/bin
    - run:
        name: Configure environment
        command: |
          vendor/bin/robo memcache:install
          vendor/bin/robo circleci:override-php-ini
          vendor/bin/robo circleci:override-drupal-settings
    - run:
        name: Install Ukids 7 database
        command: |
          vendor/bin/robo database:download-ukids7
          vendor/bin/drush sql-drop --database=ukids7 --yes
          vendor/bin/drush sql-cli --database=ukids7 < ukids7.sql
    - run:
        name: Install Ukids config database in default database
        command: |
          mysqldump -h127.0.0.1 -uroot -proot ukids8_config > ukids8_config.sql
          vendor/bin/drush sql-drop --yes
          vendor/bin/drush sql-cli < ukids8_config.sql
    - run:
        name: Run content migration
        command: |
          vendor/bin/robo local:update --no-check-outdated
          vendor/bin/robo migrate:content
    - run:
        name: Dump databases and build image
        command: |
          mv ukids7.sql scripts/database/dumps/ukids7.sql
          mv ukids8_config.sql scripts/database/dumps/ukids8_config.sql
          vendor/bin/drush sql-dump > scripts/database/dumps/ukids8.sql
          mysqldump -h127.0.0.1 -uroot -proot ukids8_initial > scripts/database/dumps/ukids8_initial.sql
          cd scripts/database
          docker login -u="$QUAY_USERNAME_V2" -p="$QUAY_PASSWORD_V2" quay.io
          docker build --tag quay.io/some-org/repository:latest .
          docker push quay.io/some-org/repository:latest
    - save_cache: *save_cache

# Declare all of the jobs we should run.
jobs:
  run-composer-install:
    <<: *composer_install
  run-migrate:
    <<: *migrate

# Declare a workflow that the jobs.
workflows:
  version: 2
  migration:
    triggers:
      - schedule:
          cron: "0 0,3,6,9,12,15,18,21 * * *"
          filters:
            branches:
              only:
                - master
    jobs:
      - run-composer-install
      - run-migrate:
          requires:
            - run-composer-install

RoboFile.php

<?php

/**
 * Command line tasks.
 *
 * @codingStandardsIgnoreStart
 * @codeCoverageIgnore
 * @SuppressWarnings(PHPMD)
 */
class RoboFile extends \Robo\Tasks {

  /**
   * Update a local environment.
   *
   * Useful after switching/pulling branches, or importing a database.
   *
   * @param array $options
   *   Associative array of command line options with their defaults.
   */
  public function localUpdate(array $options = ['no-composer' => FALSE, 'no-check-outdated' => FALSE]) {
    $collection = $this->collectionBuilder();

    $needs_outdated_check = !$options['no-check-outdated'] && $this->needsOutdatedCheck();
    $tasks = $this->localUpdateTask(!$options['no-composer'], $needs_outdated_check);

    $collection->addTaskList($tasks)->run();

    // Update stored hash for next run.
    if ($needs_outdated_check) {
      $this->updateComposerLastOutdatedCheck();
    }
  }

  /**
   * Build the tasks to update a local environment.
   *
   * @param bool $composer
   *   Whether to run composer install.
   * @param bool $check_outdated
   *   Whether to check outdated packages via composer.
   *
   * @return array
   *   Tasks to run.
   */
  protected function localUpdateTask($composer = TRUE, $check_outdated = TRUE) {
    $tasks = [];

    // Install composer dependencies.
    if ($composer) {
      $tasks[] = $this->composerInstall();
      if ($check_outdated) {
        $tasks[] = $this->checkOutdatedDependencies();
      }
    }

    // Run database updates.
    $tasks[] = $this->drush()->args('updatedb')->option('-vvv');

    // Import config.
    $tasks[] = $this->drush()->args('config-import');

    // Import config again because sometimes Drupal can't get enough of it :-D
    $tasks[] = $this->drush()->args('config-import');

    // Rebuild the cache one last time.
    $tasks[] = $this->drush()->args('cache:rebuild');

    return $tasks;
  }

  /**
   * Installs composer dependencies.
   *
   * @return \Robo\Contract\TaskInterface
   *   A task instance.
   */
  public function composerInstall() {
    $collection = $this->collectionBuilder();
    $collection->addTask(
      $this->taskComposerValidate()
      ->noCheckPublish()
    );

    $collection->addTask(
      $this->taskComposerInstall()
        ->noInteraction()
        ->envVars(['COMPOSER_ALLOW_SUPERUSER' => 1, 'COMPOSER_DISCARD_CHANGES' => 1] + getenv())
        ->optimizeAutoloader()
    );

    return $collection;
  }

  /**
   * Runs a Drush command.
   *
   * @return \Robo\Task\Base\Exec
   *   A Drush exec command.
   */
  protected function drush() {
    $drush = 'vendor/bin/drush';
    $drush = $this->taskExec($drush)->option('yes');

    return $drush;
  }

  /**
   * Get the absolute path to the docroot.
   *
   * @return string
   */
  protected function getDocroot() {
    $docroot = (getcwd()) . '/web';
    return $docroot;
  }

  /**
   * @return \Robo\Task\Base\Exec
   */
  protected function checkOutdatedDependencies() {
    return $this->taskExec('composer outdated -oD')
      ->envVars([
          'COLUMNS' => 120,
          'COMPOSER_ALLOW_SUPERUSER' => 1,
          'COMPOSER_DISCARD_CHANGES' => 1
        ] + getenv());
  }

  /**
   * Override settings.php.
   */
  public function circleciOverrideDrupalSettings() {
    $this->taskFilesystemStack()
      ->copy('.circleci/config/settings.local.php',
       'web/sites/default/settings.local.php', true)
      ->run();
  }

  /**
   * Overrides the default PHP configuration.
   */
  public function circleciOverridePhpIni() {
    $this->taskFilesystemStack()
         ->copy('.circleci/config/php-cli.ini', '/usr/local/etc/php/php-cli.ini', TRUE)
         ->run();
  }

  /**
   * Installs memcache php extension.
   *
   * @return \Robo\Result
   *   The result of the collection of tasks.
   */
  public function memcacheInstall() {
    $collection = $this->collectionBuilder();

    $collection->addTask($this->taskExec('apt-get -y install libmemcached11 libmemcachedutil2 libmemcached-dev \
          && cd /usr/local/share \
          && git clone --branch php7 https://github.com/php-memcached-dev/php-memcached \
          && cd php-memcached \
          && phpize \
          && ./configure \
          && make \
          && echo "extension=/usr/local/share/php-memcached/modules/memcached.so" > /usr/local/etc/php/conf.d/memcached.ini'));

    return $collection->run();
  }

  /**
   * Creates and configures the site database.
   *
   * @return array
   */
  protected function databaseCreateTasks(): array {
    $tasks = [];
    $tasks[] = $this->drush()->arg('sql-create')->option('debug');
    // Adjust max_allowed_packet, else the default of 16M won't allow importing.
    $tasks[] = $this->drush()->arg('sql:query')->arg('SET GLOBAL max_allowed_packet = 67108864;');
    return $tasks;
  }

  /**
   * Downloads the source database from Dropbox.
   *
   * @return \Robo\Result
   *   The result of the collection of tasks.
   */
  public function databaseDownloadUkids7() {
    $collection = $this->collectionBuilder();

    $collection->addTask($this->dropboxDownload('dump.sql.gz', 'ukids7.sql.gz'));
    $collection->addTask($this->taskExec('gunzip ukids7.sql.gz'));

    return $collection->run();
  }

  /**
   * Command to run the content migration.
   *
   * @return \Robo\Result
   *   The result of the collection of tasks.
   */
  public function migrateContent() {
    $collection = $this->collectionBuilder();
    $collection->addTask($this->drush()
      ->arg('migrate:import')
      ->option('execute-dependencies')
      ->option('feedback', '1000')
      ->option('tag', 'Content')
      ->option('-vvv'));
    $collection->addTask($this->runMediaThumbnailQueue());
    $collection->addTask($this->emptyPurgeQueue());
    return $collection->run();
  }

  /**
   * Command to recreate migrations.
   *
   * @return \Robo\Result
   *   The result of the collection of tasks.
   */
  public function migrateRecreateMigrations() {
    $collection = $this->collectionBuilder();
    $collection->addTask($this->drush()
      ->arg('migrate:delete-group')
      ->arg('migrate_drupal_7'));
    $collection->addTask($this->drush()
      ->arg('migrate:upgrade')
      ->option('legacy-db-key', 'ukids7')
      ->option('legacy-root', 'sites/default/files')
      ->option('configure-only'));
    $collection->addTask($this->drush()
      ->arg('config:export'));
    $collection->progressMessage('Review exported migrations. Do not commit the ones whose only change is the UUID.');

    return $collection->run();
  }

  /**
   * Command to re-run field migrations.
   *
   * @return \Robo\Result
   *   The result of the collection of tasks.
   */
  public function migrateRerunFieldMigrations() {
    $migrations = [
      'upgrade_d7_field',
      'upgrade_d7_field_instance',
      'upgrade_d7_field_formatter_settings',
      'upgrade_d7_field_instance_widget_settings',
      'upgrade_d7_view_modes',
    ];
    $collection = $this->collectionBuilder();
    $collection->addTask($this->drush()
      ->arg('migrate:rollback')
      ->arg(implode(',', array_reverse($migrations))));
    $collection->addTask($this->drush()
      ->arg('migrate:import')
      ->option('execute-dependencies')
      ->option('-vvv')
      ->arg(implode(',', $migrations)));
    $collection->addTask($this->drush()
      ->arg('config:export'));
    $collection->progressMessage('The resulting configuration has been exported. Review and commit the files.');

    return $collection->run();
  }

  /**
   * Migrating content will cause every node to be queued to be cleared, and
   * purge module will never be able to catch up. After 100K nodes purge
   * assumes something is wrong and stops purging entirely. This resets the
   * purge queues.
   *
   * @return \Robo\Contract\TaskInterface
   *   A task instance.
   */
  private function emptyPurgeQueue() {
    return $this->drush()->arg('p:queue:empty');
  }

  /**
   * Imports media thumbnails.
   *
   * @param int|null $thumbnails_timeout (optional) Set a timeout for ingesting thumbnails.
   *
   * @return \Robo\Contract\TaskInterface
   *   A task instance.
   */
  protected function runMediaThumbnailQueue(int $thumbnails_timeout = null) {
    $thumbnail_task = $this->drush()
      ->option('verbose')
      ->arg('queue-run')
      ->arg('media_entity_thumbnail');

    if (!empty($thumbnails_timeout)) {
      $thumbnail_task->option('time-limit', $thumbnails_timeout, '=');
    }

    return $thumbnail_task;
  }

  /**
   * Downloads a file from Dropbox to the given destination.
   *
   * @param string $filename The file to download.
   *
   * @param string $destination The destination path and file name.
   *
   * @return \Robo\Contract\TaskInterface
   *   A task instance.
   */
  protected function dropboxDownload(string $filename, string $destination) {
    return $this->taskExec('php vendor/juampynr/dropbox-api/dropbox-download.php ' . $filename . ' ' . $destination);
  }

}