Skip to content

Instantly share code, notes, and snippets.

@juampynr
Last active Oct 9, 2021
Embed
What would you like to do?
Drupal 8 common migration tasks
# CircleCI integration with Drupal 8.
# Notice: this file goes at .circleci/config.yml but Gists don't allow subdirectories.
version: 2.1
# Reusable steps.
## Defines images and working directory.
defaults: &defaults
working_directory: /var/www/html
executors:
docker_drupal8ci:
docker:
- image: juampynr/drupal8ci:latest
docker_drupal8ci_mariadb:
docker:
- image: juampynr/drupal8ci:latest
- image: quay.io/some-org/repository:latest
auth:
username: $QUAY_USERNAME_V2
password: $QUAY_PASSWORD_V2
- image: memcached:1.5
docker_drupal8ci_mariadb_selenium:
docker:
- image: juampynr/drupal8ci:latest
- image: selenium/standalone-chrome-debug:3.141.59
- image: quay.io/some-org/repository:latest
auth:
username: $QUAY_USERNAME_V2
password: $QUAY_PASSWORD_V2
- image: memcached:1.5
## Defines the cache restoring mechanism.
restore_cache: &restore_cache
# We use the composer.lock as a way to determine if we can cache our build.
keys:
- v4-dependencies-{{ .Branch }}-{{ checksum "composer.lock" }}
# fallback to using the latest master cache if no exact match is found
- v4-dependencies-master-
## Defines the cache saving mechanism.
save_cache: &save_cache
paths:
- ./vendor
- /root/.composer/cache
key: v4-dependencies-{{ .Branch }}-{{ checksum "composer.lock" }}
#Jobs
# Install composer libraries and other code dependencies.
composer_install: &composer_install
<<: *defaults
executor: docker_drupal8ci
steps:
- checkout
- restore_cache: *restore_cache
- run:
name: Configure environment
command: robo composer:install
- persist_to_workspace:
root: /var/www/html
paths: .
- save_cache: *save_cache
## Job to update the source database.
migrate: &migrate
<<: *defaults
executor: docker_drupal8ci_mariadb_selenium
steps:
- attach_workspace:
at: /var/www/html
- setup_remote_docker:
docker_layer_caching: true
- run:
name: Install Docker client
command: |
set -x
VER="18.09.3"
curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
tar -xz -C /tmp -f /tmp/docker-$VER.tgz
mv /tmp/docker/* /usr/bin
- run:
name: Configure environment
command: |
vendor/bin/robo memcache:install
vendor/bin/robo circleci:override-php-ini
vendor/bin/robo circleci:override-drupal-settings
- run:
name: Install Ukids 7 database
command: |
vendor/bin/robo database:download-ukids7
vendor/bin/drush sql-drop --database=ukids7 --yes
vendor/bin/drush sql-cli --database=ukids7 < ukids7.sql
- run:
name: Install Ukids config database in default database
command: |
mysqldump -h127.0.0.1 -uroot -proot ukids8_config > ukids8_config.sql
vendor/bin/drush sql-drop --yes
vendor/bin/drush sql-cli < ukids8_config.sql
- run:
name: Run content migration
command: |
vendor/bin/robo local:update --no-check-outdated
vendor/bin/robo migrate:content
- run:
name: Dump databases and build image
command: |
mv ukids7.sql scripts/database/dumps/ukids7.sql
mv ukids8_config.sql scripts/database/dumps/ukids8_config.sql
vendor/bin/drush sql-dump > scripts/database/dumps/ukids8.sql
mysqldump -h127.0.0.1 -uroot -proot ukids8_initial > scripts/database/dumps/ukids8_initial.sql
cd scripts/database
docker login -u="$QUAY_USERNAME_V2" -p="$QUAY_PASSWORD_V2" quay.io
docker build --tag quay.io/some-org/repository:latest .
docker push quay.io/some-org/repository:latest
- save_cache: *save_cache
# Declare all of the jobs we should run.
jobs:
run-composer-install:
<<: *composer_install
run-migrate:
<<: *migrate
# Declare a workflow that the jobs.
workflows:
version: 2
migration:
triggers:
- schedule:
cron: "0 0,3,6,9,12,15,18,21 * * *"
filters:
branches:
only:
- master
jobs:
- run-composer-install
- run-migrate:
requires:
- run-composer-install
<?php
/**
* Command line tasks.
*
* @codingStandardsIgnoreStart
* @codeCoverageIgnore
* @SuppressWarnings(PHPMD)
*/
class RoboFile extends \Robo\Tasks {
/**
* Update a local environment.
*
* Useful after switching/pulling branches, or importing a database.
*
* @param array $options
* Associative array of command line options with their defaults.
*/
public function localUpdate(array $options = ['no-composer' => FALSE, 'no-check-outdated' => FALSE]) {
$collection = $this->collectionBuilder();
$needs_outdated_check = !$options['no-check-outdated'] && $this->needsOutdatedCheck();
$tasks = $this->localUpdateTask(!$options['no-composer'], $needs_outdated_check);
$collection->addTaskList($tasks)->run();
// Update stored hash for next run.
if ($needs_outdated_check) {
$this->updateComposerLastOutdatedCheck();
}
}
/**
* Build the tasks to update a local environment.
*
* @param bool $composer
* Whether to run composer install.
* @param bool $check_outdated
* Whether to check outdated packages via composer.
*
* @return array
* Tasks to run.
*/
protected function localUpdateTask($composer = TRUE, $check_outdated = TRUE) {
$tasks = [];
// Install composer dependencies.
if ($composer) {
$tasks[] = $this->composerInstall();
if ($check_outdated) {
$tasks[] = $this->checkOutdatedDependencies();
}
}
// Run database updates.
$tasks[] = $this->drush()->args('updatedb')->option('-vvv');
// Import config.
$tasks[] = $this->drush()->args('config-import');
// Import config again because sometimes Drupal can't get enough of it :-D
$tasks[] = $this->drush()->args('config-import');
// Rebuild the cache one last time.
$tasks[] = $this->drush()->args('cache:rebuild');
return $tasks;
}
/**
* Installs composer dependencies.
*
* @return \Robo\Contract\TaskInterface
* A task instance.
*/
public function composerInstall() {
$collection = $this->collectionBuilder();
$collection->addTask(
$this->taskComposerValidate()
->noCheckPublish()
);
$collection->addTask(
$this->taskComposerInstall()
->noInteraction()
->envVars(['COMPOSER_ALLOW_SUPERUSER' => 1, 'COMPOSER_DISCARD_CHANGES' => 1] + getenv())
->optimizeAutoloader()
);
return $collection;
}
/**
* Runs a Drush command.
*
* @return \Robo\Task\Base\Exec
* A Drush exec command.
*/
protected function drush() {
$drush = 'vendor/bin/drush';
$drush = $this->taskExec($drush)->option('yes');
return $drush;
}
/**
* Get the absolute path to the docroot.
*
* @return string
*/
protected function getDocroot() {
$docroot = (getcwd()) . '/web';
return $docroot;
}
/**
* @return \Robo\Task\Base\Exec
*/
protected function checkOutdatedDependencies() {
return $this->taskExec('composer outdated -oD')
->envVars([
'COLUMNS' => 120,
'COMPOSER_ALLOW_SUPERUSER' => 1,
'COMPOSER_DISCARD_CHANGES' => 1
] + getenv());
}
/**
* Override settings.php.
*/
public function circleciOverrideDrupalSettings() {
$this->taskFilesystemStack()
->copy('.circleci/config/settings.local.php',
'web/sites/default/settings.local.php', true)
->run();
}
/**
* Overrides the default PHP configuration.
*/
public function circleciOverridePhpIni() {
$this->taskFilesystemStack()
->copy('.circleci/config/php-cli.ini', '/usr/local/etc/php/php-cli.ini', TRUE)
->run();
}
/**
* Installs memcache php extension.
*
* @return \Robo\Result
* The result of the collection of tasks.
*/
public function memcacheInstall() {
$collection = $this->collectionBuilder();
$collection->addTask($this->taskExec('apt-get -y install libmemcached11 libmemcachedutil2 libmemcached-dev \
&& cd /usr/local/share \
&& git clone --branch php7 https://github.com/php-memcached-dev/php-memcached \
&& cd php-memcached \
&& phpize \
&& ./configure \
&& make \
&& echo "extension=/usr/local/share/php-memcached/modules/memcached.so" > /usr/local/etc/php/conf.d/memcached.ini'));
return $collection->run();
}
/**
* Creates and configures the site database.
*
* @return array
*/
protected function databaseCreateTasks(): array {
$tasks = [];
$tasks[] = $this->drush()->arg('sql-create')->option('debug');
// Adjust max_allowed_packet, else the default of 16M won't allow importing.
$tasks[] = $this->drush()->arg('sql:query')->arg('SET GLOBAL max_allowed_packet = 67108864;');
return $tasks;
}
/**
* Downloads the source database from Dropbox.
*
* @return \Robo\Result
* The result of the collection of tasks.
*/
public function databaseDownloadUkids7() {
$collection = $this->collectionBuilder();
$collection->addTask($this->dropboxDownload('dump.sql.gz', 'ukids7.sql.gz'));
$collection->addTask($this->taskExec('gunzip ukids7.sql.gz'));
return $collection->run();
}
/**
* Command to run the content migration.
*
* @return \Robo\Result
* The result of the collection of tasks.
*/
public function migrateContent() {
$collection = $this->collectionBuilder();
$collection->addTask($this->drush()
->arg('migrate:import')
->option('execute-dependencies')
->option('feedback', '1000')
->option('tag', 'Content')
->option('-vvv'));
$collection->addTask($this->runMediaThumbnailQueue());
$collection->addTask($this->emptyPurgeQueue());
return $collection->run();
}
/**
* Command to recreate migrations.
*
* @return \Robo\Result
* The result of the collection of tasks.
*/
public function migrateRecreateMigrations() {
$collection = $this->collectionBuilder();
$collection->addTask($this->drush()
->arg('migrate:delete-group')
->arg('migrate_drupal_7'));
$collection->addTask($this->drush()
->arg('migrate:upgrade')
->option('legacy-db-key', 'ukids7')
->option('legacy-root', 'sites/default/files')
->option('configure-only'));
$collection->addTask($this->drush()
->arg('config:export'));
$collection->progressMessage('Review exported migrations. Do not commit the ones whose only change is the UUID.');
return $collection->run();
}
/**
* Command to re-run field migrations.
*
* @return \Robo\Result
* The result of the collection of tasks.
*/
public function migrateRerunFieldMigrations() {
$migrations = [
'upgrade_d7_field',
'upgrade_d7_field_instance',
'upgrade_d7_field_formatter_settings',
'upgrade_d7_field_instance_widget_settings',
'upgrade_d7_view_modes',
];
$collection = $this->collectionBuilder();
$collection->addTask($this->drush()
->arg('migrate:rollback')
->arg(implode(',', array_reverse($migrations))));
$collection->addTask($this->drush()
->arg('migrate:import')
->option('execute-dependencies')
->option('-vvv')
->arg(implode(',', $migrations)));
$collection->addTask($this->drush()
->arg('config:export'));
$collection->progressMessage('The resulting configuration has been exported. Review and commit the files.');
return $collection->run();
}
/**
* Migrating content will cause every node to be queued to be cleared, and
* purge module will never be able to catch up. After 100K nodes purge
* assumes something is wrong and stops purging entirely. This resets the
* purge queues.
*
* @return \Robo\Contract\TaskInterface
* A task instance.
*/
private function emptyPurgeQueue() {
return $this->drush()->arg('p:queue:empty');
}
/**
* Imports media thumbnails.
*
* @param int|null $thumbnails_timeout (optional) Set a timeout for ingesting thumbnails.
*
* @return \Robo\Contract\TaskInterface
* A task instance.
*/
protected function runMediaThumbnailQueue(int $thumbnails_timeout = null) {
$thumbnail_task = $this->drush()
->option('verbose')
->arg('queue-run')
->arg('media_entity_thumbnail');
if (!empty($thumbnails_timeout)) {
$thumbnail_task->option('time-limit', $thumbnails_timeout, '=');
}
return $thumbnail_task;
}
/**
* Downloads a file from Dropbox to the given destination.
*
* @param string $filename The file to download.
*
* @param string $destination The destination path and file name.
*
* @return \Robo\Contract\TaskInterface
* A task instance.
*/
protected function dropboxDownload(string $filename, string $destination) {
return $this->taskExec('php vendor/juampynr/dropbox-api/dropbox-download.php ' . $filename . ' ' . $destination);
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment