Skip to content

Instantly share code, notes, and snippets.

@Renrhaf

Renrhaf/AFOUND-731.patch

Created July 25, 2016 08:43
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Renrhaf/0820fa202cb33211a842ca0f8e117676 to your computer and use it in GitHub Desktop.
Save Renrhaf/0820fa202cb33211a842ca0f8e117676 to your computer and use it in GitHub Desktop.
Download ZIP
AFOUND-731
Raw
AFOUND-731.patch
From 55334013943081fbbf94f735af8395f5faf94a51 Mon Sep 17 00:00:00 2001
From: lsantos-oq <luis.santos@openquest.pt>
Date: Thu, 19 May 2016 17:07:58 +0100
Subject: [PATCH 01/12] AFOUND-731 Initial commit
---
modules/custom/taxonomy_opa_migrate/README.md | 70 ++++
.../includes/opa_categories_reader.inc | 79 +++++
.../migrate/sources/rabbit_mq_opa_base.inc | 332 ++++++++++++++++++
.../migrate/sources/taxonomy_opa.inc | 104 ++++++
.../migrate/sources/taxonomy_rabbit_mq_opa.inc | 381 +++++++++++++++++++++
.../migrate/sources/unique_code_map.inc | 57 +++
.../migrate/taxonomy_opa_migrate.inc | 187 ++++++++++
.../taxonomy_opa_migrate.drush.inc | 143 ++++++++
.../taxonomy_opa_migrate/taxonomy_opa_migrate.info | 19 +
.../taxonomy_opa_migrate.install | 1 +
.../taxonomy_opa_migrate.module | 1 +
11 files changed, 1374 insertions(+)
create mode 100644 modules/custom/taxonomy_opa_migrate/README.md
create mode 100644 modules/custom/taxonomy_opa_migrate/includes/opa_categories_reader.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/migrate/sources/rabbit_mq_opa_base.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_rabbit_mq_opa.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/migrate/sources/unique_code_map.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
create mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.install
create mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
diff --git a/modules/custom/taxonomy_opa_migrate/README.md b/modules/custom/taxonomy_opa_migrate/README.md
new file mode 100644
index 0000000..beb2a10
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/README.md
@@ -0,0 +1,70 @@
+Taxonomy OPA Migrate
+=================
+
+Module providing support for migrating from OPA into `taxonomy terms`. Dynamic migrations use the migrate framework and Drush commands are supported to run migrations via the command-line.
+
+### Configuring OPA
+
+Follow configuration instructions in the `arte_opa` module to make sure you can connect to OPA.
+
+### Mapping
+
+See the Migrate class `taxonomy_opa.inc` for mapping information.
+
+### Sources
+
+#### Full OPA queries
+
+The default configuration is querying OPA directly with the given parameters (see "Drush commands" for the parameters to pass on).
+
+#### RabbitMQ
+
+Another method is to use RabbitMQ notifications to update taxonomy terms, given a notification that wraps the code. OPA is queried for every code.
+
+The advantage of this method is that it does not need to query OPA for full listings of videos, but instead will only add and update videos that are added to the queue, which is directly controlled by ARTE.
+
+Currently, RabbitMQ is supported and in production for ARTE Concert, using the rabbit_mq_opa Migrate source.
+
+To enable RabbitMQ, you can pass on the option `--rabbitmq` to the drush command, and you also need to configure the following variables (preferably, directly in `settings.php`)
+
+ // OPA Rabbit settings (example from ARTE Concert production)
+ $conf['scald_opa_migrate_amqp_host'] = 'ravel.arte.tv';
+ $conf['scald_opa_migrate_amqp_port'] = '5672';
+ $conf['scald_opa_migrate_amqp_user'] = 'concert';
+ $conf['scald_opa_migrate_amqp_pass'] = '*******';
+ $conf['scald_opa_migrate_amqp_vhost'] = '/mom_v2_prod';
+ $conf['scald_opa_migrate_amqp_queue'] = 'opa_updates_for_concert';
+
+### Drush commands
+
+The module provides a custom Drush command for synchronization
+
+ drush tos
+
+See `drush help sos` for examples. For example, to imports 10 videos (german) from the ALW platform
+
+ drush tos --platform=ALW --bundle=genres
+
+Note that we map different language version onto *the same atom*, meaning that after running these two commands we'll have 10 opa atoms that contain both the German and the French versions (using Drupal's Entity Translation system).
+
+## RabbitMQ server
+
+This module also supports migrating from RabbitMQ. A message is sent with enough information to query OPA for the right video resources.
+
+For local testing, you can install RabbitMQ and fill up the queue with sample data.
+
+Start testing as follows
+
+`rabbitmq-server`
+Starts the server
+
+`php test/rabbitmq/populate_queue.php`
+Populates some test data in the queue (video). To change the test data, add files to `list.php`
+
+`rabbitmqadmin list queues`
+Check how many items are in the video queue
+
+Now run the drush command to import in your site (see below).
+
+Purges the queue (emptying it)
+`rabbitmqadmin purge queue name=opa`
diff --git a/modules/custom/taxonomy_opa_migrate/includes/opa_categories_reader.inc b/modules/custom/taxonomy_opa_migrate/includes/opa_categories_reader.inc
new file mode 100644
index 0000000..67f079a
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/includes/opa_categories_reader.inc
@@ -0,0 +1,79 @@
+<?php
+
+/**
+ * @file
+ * OPA Categories reader
+ */
+
+class OPACategoriesReader extends OPAReader {
+
+ /**
+ * Initialize the members.
+ */
+ public function __construct($opa, $params) {
+ parent::__construct($opa, $params, 'code', 'categories');
+ }
+
+ /**
+ * Implementation of Iterator::next().
+ *
+ * Populates currentElement (the object being retrieved) and currentId (that
+ * object's unique identifier) from the specified JSON file. Sets both to
+ * NULL at end-of-file. Handles properly-formed JSON, as well as some improper
+ * coding (specifically that generated in Ning exports).
+ *
+ * @return void
+ */
+ public function next() {
+ // Use pagination when no user limit was specified or when the specified limit is
+ // bigger than the maximum allowed by OPA
+ $pagination = is_null($this->limit) || $this->limit > $this->pagination_limit;
+
+ // Open the file and position it if necessary
+ if (empty($this->data)) {
+ $options = $this->params;
+ $options['page'] = $this->page;
+ $options['limit'] = $pagination ? $this->pagination_limit : $this->limit;
+
+ // API method callback with pagination and limits
+ $this->data = $this->opa->apiCall($this->resourcePath(), $options);
+
+ // @todo add try/catch
+ if (!$this->data) {
+ return FALSE;
+ }
+ }
+
+ // Increment Id
+ $this->position++;
+
+ // If we're paginating with a user-specified limit, check if the iterator has reached the limit
+ //
+ // NB the last OPA page will contain the pagination limit, i.e. 50 items by default.
+ // If however the user-specified limit is 60 for instance, we only want the remaining 10 items from
+ // that last page.
+ $last_page_delta = $this->limit - (($this->page - 1) * $this->pagination_limit);
+ if (!is_null($this->limit) && $pagination && $this->position + 1 > $last_page_delta) {
+ // We are done!
+ $this->currentElement = NULL;
+ }
+ elseif (isset($this->data[$this->resource][$this->position])) {
+ $this->currentElement = $this->getData();
+ $this->currentId = $this->currentElement['code'];
+ }
+ else {
+ // Check if we need to advance the pagination
+ if (isset($this->data['meta'][$this->resource]['pages']) && $pagination && $this->page < $this->data['meta'][$this->resource]['pages']) {
+ // Yes, and we have pages available still, so increment page and reset data to fetch the next page
+ $this->page++;
+ $this->position = -1;
+ $this->data = array();
+ $this->next();
+ }
+ else {
+ // We are done!
+ $this->currentElement = NULL;
+ }
+ }
+ }
+}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/sources/rabbit_mq_opa_base.inc b/modules/custom/taxonomy_opa_migrate/migrate/sources/rabbit_mq_opa_base.inc
new file mode 100644
index 0000000..01c4b09
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/migrate/sources/rabbit_mq_opa_base.inc
@@ -0,0 +1,332 @@
+<?php
+
+// Use the AMQP namespace
+
+// This include path will only work if the scald_opa_migrate modules doesn't move within the profile.
+// @todo find a more elegant way to include the correct namespace
+require_once __DIR__ . '/../../../../../libraries/php-amqplib/vendor/autoload.php';
+use PhpAmqpLib\Connection\AMQPConnection;
+
+/**
+ * @file
+ * Define a MigrateSource for importing from APIOS using RabbitMQ
+ */
+
+/**
+ * Implementation of MigrateSource, to handle imports from Drupal connections.
+ */
+class MigrateSourceRabbitMQOPABase extends MigrateSourceOPABase {
+
+ /**
+ * The AMQP connection.
+ */
+ protected $connection = NULL;
+
+
+ /**
+ * The AMQP channel object.
+ */
+ protected $channel = NULL;
+
+ /**
+ * The AMQP queue from which to fetch items.
+ */
+ protected $queue = '';
+
+ /**
+ * List of available source fields.
+ *
+ * @var array
+ */
+ protected $fields = array();
+
+ /**
+ * Supported languages to fetch videos for.
+ *
+ * @var array
+ */
+ protected $languages = array();
+
+ /**
+ * The current message number
+ */
+ protected $currentNumber = 0;
+
+ /**
+ * The total number of messages in the queue
+ */
+ protected $totalQueue = 0;
+
+ /**
+ * The OPA source
+ */
+ protected $opa = NULL;
+
+
+ /**
+ * Additional OPA source parameters
+ */
+ protected $params = array();
+
+
+ /**
+ * The mini-queue. Everytime we do a callback to OPA we might return a series
+ * of videos in one go (e.g. all translations of a single video), these are then queued
+ * up here and processed.
+ */
+ protected $miniQueue = array();
+
+ /**
+ * Return a string representing AMQP
+ */
+ public function __toString() {
+ return "AMQP " . $this->queue;
+ }
+
+ /**
+ * Connect to AMQP.
+ */
+ protected function connect() {
+ if (!isset($this->channel)) {
+ $this->connection = new AMQPConnection(
+ $this->configuration['host'],
+ $this->configuration['port'],
+ $this->configuration['user'],
+ $this->configuration['pass'],
+ $this->configuration['vhost']
+ );
+ $this->channel = $this->connection->channel();
+ $this->channel->queue_declare($this->queue, TRUE, TRUE);
+ }
+
+ if ($this->channel->active) {
+ return TRUE;
+ }
+ else {
+ throw new Exception(t("AMQP Channel is not active for queue !q", array('!q' => $this->queue)));
+ return FALSE;
+ }
+ }
+
+ /**
+ * Disconnect from AMQP
+ */
+ protected function disconnect() {
+ $this->channel->close();
+ $this->connection->close();
+ }
+
+ /**
+ * Simple initialization.
+ */
+ public function __construct(array $configuration, $queue, $opa, $params, $languages, $fields = array(), array $options = array()) {
+ parent::__construct($options);
+
+ // Load the PHP AMQP library
+ libraries_load('php-amqplib');
+
+ $this->queue = $queue;
+ $this->configuration = $configuration;
+ $this->languages = $languages;
+ $this->fields = $fields;
+ $this->params = $params;
+
+ $this->connect();
+
+ // Set the number of messages in the queue
+ $this->totalQueue = $this->computeCount();
+
+ // Save OPA source
+ $this->opa = $opa;
+ }
+
+
+ function __destruct() {
+ // Disconnect
+ // @todo research why disconnecting doesn't work, it throws errors
+ // $this->disconnect();
+ }
+
+ /**
+ * Returns a list of fields available to be mapped from the source query.
+ *
+ * @return array
+ * Keys: machine names of the fields (to be passed to addFieldMapping)
+ * Values: Human-friendly descriptions of the fields.
+ */
+ public function fields() {
+ return $this->fields;
+ }
+
+ /**
+ * Return a count of all available source records.
+ */
+ public function computeCount() {
+ if ($this->channel) {
+ $ret = $this->channel->queue_declare($this->queue, TRUE, TRUE);
+ if ($ret && isset($ret[1])) {
+ return $ret[1];
+ }
+ else {
+ return 0;
+ }
+ }
+ else {
+ // Queue isn't alive?
+ return 0;
+ }
+ }
+
+ /**
+ * Implementation of MigrateSource::performRewind().
+ *
+ * @return void
+ */
+ public function performRewind() {
+ // Nothing for now
+ }
+
+ /**
+ * Implementation of MigrateSource::getNextRow().
+ *
+ * @return null|stdClass
+ */
+ public function getNextRow() {
+ // First off-load any items from our mini queue.
+ if (!empty($this->miniQueue)) {
+ $return = array_shift($this->miniQueue);
+ return $return;
+ }
+
+ // Make sure we are connected to Rabbit
+ if (!$this->channel) {
+ return NULL;
+ }
+
+ // Try fetching new messages from the queue now
+ try {
+ $msg = $this->channel->basic_get($this->queue);
+ if ($msg) {
+ $items = $this->fetchCategories($msg->body, $this->languages);
+ $migration = Migration::currentMigration();
+
+ // We received the videos correctly
+ $this->currentNumber++;
+
+ // Acknowledge message reception
+ if (!variable_get('scald_opa_migrate_rabbitmq_disable_acknowledgement', FALSE)) {
+ $this->channel->basic_ack($msg->delivery_info['delivery_tag']);
+ $migration->displayMessage(
+ format_string(
+ 'Message !num/!total received from queue !queue',
+ array(
+ '!num' => $this->currentNumber,
+ '!total' => $this->totalQueue,
+ '!queue' => $this->queue,
+ )
+ ),
+ 'status'
+ );
+ }
+
+ // Add videos to our mini-queue so they can be processed
+ $this->miniQueue = $items['videos'];
+ return $this->getNextRow();
+ }
+ }
+ // Catch OPA exceptions but continue loading off the queue
+ catch (OPAException $e) {
+ $migration = Migration::currentMigration();
+
+ // Something went wrong in fetching videos from OPA, report
+ $migration = Migration::currentMigration();
+ $message = t('Loading of OPA JSON from the AQMP queue failed. Acknowledging message and moving on. Error message from OPA: !message. Original API call made to "!url". ', array('!message' => $e->getMessage(), '!url' => $e->getUrl()));
+
+ $migration->getMap()->saveMessage(array($this->getCurrentKey()), $message, MigrationBase::MESSAGE_ERROR);
+
+ // Acknowledge receival, even though we couldn't find anything in OPA
+ $this->channel->basic_ack($msg->delivery_info['delivery_tag']);
+
+ $migration->displayMessage(
+ format_string(
+ 'Message !num/!total received from queue !queue but not processed',
+ array(
+ '!num' => $this->currentNumber,
+ '!total' => $this->totalQueue,
+ '!queue' => $this->queue,
+ )
+ ),
+ 'error'
+ );
+
+ return $this->getNextRow();
+ }
+ catch (Exception $e) {
+ $migration = Migration::currentMigration();
+ $message = t('Error getting item from the AMQP queue: %exp_message<br/><br/>%exp_trace', array('%exp_message' => $e->getMessage(), '%exp_trace' => $e->getTraceAsString()));
+ $migration->getMap()->saveMessage(
+ array($this->getCurrentKey()), $message, MigrationBase::MESSAGE_ERROR);
+
+ return NULL;
+ }
+ }
+
+ /**
+ * Fetch an OPA reader for a given program ID.
+ *
+ * Even though this function is really simple, it gets separated into
+ * its own function to make it easier to write test classes around OPA.
+ */
+ protected function getOPAReader($program_id) {
+ $reader = new OPACategoriesReader($this->opa, array(
+ 'programId' => $program_id
+ ) + $this->params);
+
+ return $reader;
+ }
+
+ /**
+ * Implementation of Iterator::next() - subclasses of MigrateSource should
+ * implement getNextRow() to retrieve the next valid source rocord to process.
+ *
+ * Adaptation to make sure we traverse every single line regardless
+ * of whether we've imported this one before. Necessary since the same video can be added
+ * twice.
+ */
+ public function next() {
+ $this->currentKey = NULL;
+ $this->currentRow = NULL;
+ migrate_instrument_start(get_class($this) . ' getNextRow');
+ while ($row = $this->getNextRow()) {
+ migrate_instrument_stop(get_class($this) . ' getNextRow');
+ // Populate the source key for this row
+ $this->currentKey = $this->activeMigration->prepareKey(
+ $this->activeMap->getSourceKey(), $row);
+
+ // Pick up the existing map row, if any, unless getNextRow() did it.
+ if (!$this->mapRowAdded) {
+ $map_row = $this->activeMap->getRowBySource($this->currentKey);
+ // Add map info to the row, if present
+ if ($map_row) {
+ foreach ($map_row as $field => $value) {
+ $field = 'migrate_map_' . $field;
+ $row->$field = $value;
+ }
+ }
+ }
+
+ if ($this->prepareRow($row) !== FALSE) {
+ // Finally, we've got a keeper.
+ $this->currentRow = $row;
+ break;
+ }
+ else {
+ $this->currentRow = NULL;
+ }
+ }
+ migrate_instrument_stop(get_class($this) . ' getNextRow');
+ if (!$this->currentRow) {
+ $this->currentKey = NULL;
+ }
+ }
+}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc b/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc
new file mode 100644
index 0000000..e07ff45
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc
@@ -0,0 +1,104 @@
+<?php
+
+/**
+ * @file
+ * Support for the OPA source, accepting an OPA class and parameters to make
+ * connections.
+ */
+class MigrateSourceTaxonomyOPA extends MigrateSource {
+
+ protected $reader;
+
+ /**
+ * Source constructor.
+ */
+ public function __construct($opa, $params, array $fields = array(), array $options = array()) {
+ parent::__construct($options);
+
+ $this->fields = $fields;
+
+ // Setup our reader
+ $this->reader = new OPACategoriesReader($opa, $params);
+ }
+
+ /**
+ * Return a string representing OPA
+ */
+ public function __toString() {
+ return (string) $this->reader;
+ }
+
+ /**
+ * Returns a list of fields available to be mapped from the source query.
+ *
+ * @return array
+ * Keys: machine names of the fields (to be passed to addFieldMapping)
+ * Values: Human-friendly descriptions of the fields.
+ */
+ public function fields() {
+ return $this->fields;
+ }
+
+ /**
+ * Return a count of all available source records.
+ */
+ public function computeCount() {
+ try {
+ return count($this->reader);
+ }
+ catch (OPAException $e) {
+ // @todo log somewhere?
+ return 0;
+ }
+ }
+
+ /**
+ * Implementation of MigrateSource::performRewind().
+ */
+ public function performRewind() {
+ $this->reader->rewind();
+ }
+
+ /**
+ * Implementation of MigrationSource::getNextRow().
+ *
+ * @return stdClass
+ * data for the next row from the JSON source files
+ */
+ public function getNextRow() {
+ migrate_instrument_start('MigrateSourceOPA::next');
+
+ static $next = NULL;
+
+ if (!isset($next) && $this->reader->valid()) {
+ $next = (object)$this->reader->current();
+ }
+
+ if (!isset($next)) {
+ return;
+ }
+
+ $row = $next;
+ do {
+ $this->reader->next();
+
+ if ($this->reader->valid()) {
+ $next = (object)$this->reader->current();
+
+ // Keep looping on our results if $row and $next have the
+ // same primary key, checking if there's more for that
+ // program in the queue.
+ $same = ($row->code == $next->code && $row->kind == $next->kind && $row->language == $next->language);
+ }
+ else {
+ $next = NULL;
+ $same = FALSE;
+ }
+ }
+ while ($same && $next);
+
+ migrate_instrument_stop('MigrateSourceOPA::next');
+
+ return $row;
+ }
+}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_rabbit_mq_opa.inc b/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_rabbit_mq_opa.inc
new file mode 100644
index 0000000..5978084
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_rabbit_mq_opa.inc
@@ -0,0 +1,381 @@
+<?php
+
+// Use the AMQP namespace
+
+// This include path will only work if the scald_opa_migrate modules doesn't move within the profile.
+// @todo find a more elegant way to include the correct namespace
+require_once __DIR__ . '/../../../../../libraries/php-amqplib/vendor/autoload.php';
+use PhpAmqpLib\Connection\AMQPConnection;
+
+/**
+ * @file
+ * Define a MigrateSource for importing from APIOS using RabbitMQ
+ */
+
+/**
+ * Implementation of MigrateSource, to handle imports from Drupal connections.
+ */
+class MigrateSourceRabbitMQTaxonomyOPA extends MigrateSourceOPABase {
+
+ /**
+ * The AMQP connection.
+ */
+ protected $connection = NULL;
+
+
+ /**
+ * The AMQP channel object.
+ */
+ protected $channel = NULL;
+
+ /**
+ * The AMQP queue from which to fetch items.
+ */
+ protected $queue = '';
+
+ /**
+ * List of available source fields.
+ *
+ * @var array
+ */
+ protected $fields = array();
+
+ /**
+ * Supported languages to fetch videos for.
+ *
+ * @var array
+ */
+ protected $languages = array();
+
+ /**
+ * The current message number
+ */
+ protected $currentNumber = 0;
+
+ /**
+ * The total number of messages in the queue
+ */
+ protected $totalQueue = 0;
+
+ /**
+ * The OPA source
+ */
+ protected $opa = NULL;
+
+
+ /**
+ * Additional OPA source parameters
+ */
+ protected $params = array();
+
+
+ /**
+ * The mini-queue. Everytime we do a callback to OPA we might return a series
+ * of videos in one go (e.g. all translations of a single video), these are then queued
+ * up here and processed.
+ */
+ protected $miniQueue = array();
+
+ /**
+ * Return a string representing AMQP
+ */
+ public function __toString() {
+ return "AMQP " . $this->queue;
+ }
+
+ /**
+ * Connect to AMQP.
+ */
+ protected function connect() {
+ if (!isset($this->channel)) {
+ $this->connection = new AMQPConnection(
+ $this->configuration['host'],
+ $this->configuration['port'],
+ $this->configuration['user'],
+ $this->configuration['pass'],
+ $this->configuration['vhost']
+ );
+ $this->channel = $this->connection->channel();
+ $this->channel->queue_declare($this->queue, TRUE, TRUE);
+ }
+
+ if ($this->channel->active) {
+ return TRUE;
+ }
+ else {
+ throw new Exception(t("AMQP Channel is not active for queue !q", array('!q' => $this->queue)));
+ return FALSE;
+ }
+ }
+
+ /**
+ * Disconnect from AMQP
+ */
+ protected function disconnect() {
+ $this->channel->close();
+ $this->connection->close();
+ }
+
+ /**
+ * Simple initialization.
+ */
+ public function __construct(array $configuration, $queue, $opa, $params, $languages, $fields = array(), array $options = array()) {
+ parent::__construct($options);
+
+ // Load the PHP AMQP library
+ libraries_load('php-amqplib');
+
+ $this->queue = $queue;
+ $this->configuration = $configuration;
+ $this->languages = $languages;
+ $this->fields = $fields;
+ $this->params = $params;
+
+ $this->connect();
+
+ // Set the number of messages in the queue
+ $this->totalQueue = $this->computeCount();
+
+ // Save OPA source
+ $this->opa = $opa;
+ }
+
+
+ function __destruct() {
+ // Disconnect
+ // @todo research why disconnecting doesn't work, it throws errors
+ // $this->disconnect();
+ }
+
+ /**
+ * Returns a list of fields available to be mapped from the source query.
+ *
+ * @return array
+ * Keys: machine names of the fields (to be passed to addFieldMapping)
+ * Values: Human-friendly descriptions of the fields.
+ */
+ public function fields() {
+ return $this->fields;
+ }
+
+ /**
+ * Return a count of all available source records.
+ */
+ public function computeCount() {
+ if ($this->channel) {
+ $ret = $this->channel->queue_declare($this->queue, TRUE, TRUE);
+ if ($ret && isset($ret[1])) {
+ return $ret[1];
+ }
+ else {
+ return 0;
+ }
+ }
+ else {
+ // Queue isn't alive?
+ return 0;
+ }
+ }
+
+ /**
+ * Implementation of MigrateSource::performRewind().
+ *
+ * @return void
+ */
+ public function performRewind() {
+ // Nothing for now
+ }
+
+ /**
+ * Implementation of MigrateSource::getNextRow().
+ *
+ * @return null|stdClass
+ */
+ public function getNextRow() {
+ // First off-load any items from our mini queue.
+ if (!empty($this->miniQueue)) {
+ $return = array_shift($this->miniQueue);
+ return $return;
+ }
+
+ // Make sure we are connected to Rabbit
+ if (!$this->channel) {
+ return NULL;
+ }
+
+ // Try fetching new messages from the queue now
+ try {
+ $msg = $this->channel->basic_get($this->queue);
+ if ($msg) {
+ $items = $this->fetchCategories($msg->body, $this->languages);
+ $migration = Migration::currentMigration();
+
+ // We received the videos correctly
+ $this->currentNumber++;
+
+ // Acknowledge message reception
+ if (!variable_get('scald_opa_migrate_rabbitmq_disable_acknowledgement', FALSE)) {
+ $this->channel->basic_ack($msg->delivery_info['delivery_tag']);
+ $migration->displayMessage(
+ format_string(
+ 'Message !num/!total received from queue !queue',
+ array(
+ '!num' => $this->currentNumber,
+ '!total' => $this->totalQueue,
+ '!queue' => $this->queue,
+ )
+ ),
+ 'status'
+ );
+ }
+
+ // Add videos to our mini-queue so they can be processed
+ $this->miniQueue = $items['videos'];
+ return $this->getNextRow();
+ }
+ }
+ // Catch OPA exceptions but continue loading off the queue
+ catch (OPAException $e) {
+ $migration = Migration::currentMigration();
+
+ // Something went wrong in fetching videos from OPA, report
+ $migration = Migration::currentMigration();
+ $message = t('Loading of OPA JSON from the AQMP queue failed. Acknowledging message and moving on. Error message from OPA: !message. Original API call made to "!url". ', array('!message' => $e->getMessage(), '!url' => $e->getUrl()));
+
+ $migration->getMap()->saveMessage(array($this->getCurrentKey()), $message, MigrationBase::MESSAGE_ERROR);
+
+ // Acknowledge receival, even though we couldn't find anything in OPA
+ $this->channel->basic_ack($msg->delivery_info['delivery_tag']);
+
+ $migration->displayMessage(
+ format_string(
+ 'Message !num/!total received from queue !queue but not processed',
+ array(
+ '!num' => $this->currentNumber,
+ '!total' => $this->totalQueue,
+ '!queue' => $this->queue,
+ )
+ ),
+ 'error'
+ );
+
+ return $this->getNextRow();
+ }
+ catch (Exception $e) {
+ $migration = Migration::currentMigration();
+ $message = t('Error getting item from the AMQP queue: %exp_message<br/><br/>%exp_trace', array('%exp_message' => $e->getMessage(), '%exp_trace' => $e->getTraceAsString()));
+ $migration->getMap()->saveMessage(
+ array($this->getCurrentKey()), $message, MigrationBase::MESSAGE_ERROR);
+
+ return NULL;
+ }
+ }
+
+ /**
+ * Fetch an OPA reader for a given program ID.
+ *
+ * Even though this function is really simple, it gets separated into
+ * its own function to make it easier to write test classes around OPA.
+ */
+ protected function getOPAReader($program_id) {
+ $reader = new OPACategoriesReader($this->opa, array(
+ 'programId' => $program_id
+ ) + $this->params);
+
+ return $reader;
+ }
+
+ /**
+ * Fetch videos from a rabbit message.
+ *
+ * This will use the programId of the Rabbit message, and query OPA to fetch
+ * informations about that program. Each program ID can be found both in
+ * multiple languages and multiple platforms ; for each language, only find
+ * the video that's best suited for use on the website.
+ *
+ * @param [type] $msg [description]
+ * @param [type] $languages Array of supported language codes we should consider.
+ * @return [type] [description]
+ */
+ protected function fetchCategories($msg, $languages) {
+ $item = drupal_json_decode($msg);
+
+ // Load up the right item now
+ $reader = $this->getOPAReader($item['programId']);
+
+ $categories = array(
+ 'programId' => $item['programId'],
+ 'categories' => array(),
+ );
+
+ foreach ($reader as $key => $value) {
+ // Only process items in the supported languages
+ if (in_array($value['language'], $languages)) {
+ // Convert to an object
+ $row = new stdClass();
+ foreach ($value as $k => $v) {
+ $row->$k = $v;
+ }
+
+ if (empty($categories['categories'][$row->language])) {
+ $categories['categories'][$row->language] = $row;
+ }
+ else {
+ // Check if this video is better suited for importing
+ // than the other one in the same language returned by
+ // OPA.
+ $original_row = $categories['categories'][$row->language];
+ $categories['categories'][$row->language] = $this->pickBestSuitedRecord($row, $original_row);
+ }
+ }
+ }
+
+ return $videos;
+ }
+
+
+ /**
+ * Implementation of Iterator::next() - subclasses of MigrateSource should
+ * implement getNextRow() to retrieve the next valid source rocord to process.
+ *
+ * Adaptation to make sure we traverse every single line regardless
+ * of whether we've imported this one before. Necessary since the same video can be added
+ * twice.
+ */
+ public function next() {
+ $this->currentKey = NULL;
+ $this->currentRow = NULL;
+ migrate_instrument_start(get_class($this) . ' getNextRow');
+ while ($row = $this->getNextRow()) {
+ migrate_instrument_stop(get_class($this) . ' getNextRow');
+ // Populate the source key for this row
+ $this->currentKey = $this->activeMigration->prepareKey(
+ $this->activeMap->getSourceKey(), $row);
+
+ // Pick up the existing map row, if any, unless getNextRow() did it.
+ if (!$this->mapRowAdded) {
+ $map_row = $this->activeMap->getRowBySource($this->currentKey);
+ // Add map info to the row, if present
+ if ($map_row) {
+ foreach ($map_row as $field => $value) {
+ $field = 'migrate_map_' . $field;
+ $row->$field = $value;
+ }
+ }
+ }
+
+ if ($this->prepareRow($row) !== FALSE) {
+ // Finally, we've got a keeper.
+ $this->currentRow = $row;
+ break;
+ }
+ else {
+ $this->currentRow = NULL;
+ }
+ }
+ migrate_instrument_stop(get_class($this) . ' getNextRow');
+ if (!$this->currentRow) {
+ $this->currentKey = NULL;
+ }
+ }
+}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/sources/unique_code_map.inc b/modules/custom/taxonomy_opa_migrate/migrate/sources/unique_code_map.inc
new file mode 100644
index 0000000..473c941
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/migrate/sources/unique_code_map.inc
@@ -0,0 +1,57 @@
+<?php
+
+/**
+ * @file
+ *
+ * Extends SQL map to load unique records if they already exist beyond this map.
+ * Uniqueness is being calculated using the code of the category.
+ *
+ * This can be used to update data that was previously migrated using different
+ * migrations or sources, to avoid that duplicates are being created.
+ */
+
+class MigrateUniqueCodeSQLMap extends MigrateSQLMap {
+
+ /**
+ * Retrieve a row from the map table, given a source ID
+ *
+ * @param array $source_id
+ */
+ public function getRowBySource(array $source_id) {
+ $source_id = array_shift($source_id);
+var_dump($source_id);exit;
+ migrate_instrument_start('mapRowBySource');
+ $query = $this->connection->select($this->mapTable, 'map')->fields('map');
+ foreach ($this->sourceKeyMap as $key_name) {
+ $query = $query->condition("map.$key_name", $source_id, '=');
+ }
+ $result = $query->execute();
+ $res = $result->fetchAssoc();
+
+ // This is where we change normal behavior:
+ // if nothing can be found in the map, we check if we can find this using the
+ // source ID. If we can find a nid, we pass this along and Migrate will adopt
+ // this from now on and add this to the map.
+ if (!$res && $em = ScaldOPAMigration::extractEMNumberFromKey($source_id)) {
+ $sid = scald_search(array('base_id' => $em, 'type' => 'video'), FALSE, TRUE);
+ if ($sid) {
+ // We actually found a previously imported atom with that emission number, use that atom instead
+ $res = array(
+ 'sourceid1' => $source_id,
+ 'destid1' => $sid,
+ 'needs_update' => 1,
+ 'rollback_action' => 0,
+ 'last_imported' => 0,
+ );
+
+ // Log this action
+ Migration::displayMessage(t('Used previously imported atom with sid !sid for emission number !em that was not present in mapping table.', array('!sid' => $sid, '!em' => $em)), 'warning');
+ }
+ }
+
+ migrate_instrument_stop('mapRowBySource');
+
+ // Return the result
+ return $res;
+ }
+}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
new file mode 100644
index 0000000..289bf4a
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
@@ -0,0 +1,187 @@
+<?php
+
+/**
+ * @file
+ * Migration class for Taxonomy OPA migrations.
+ */
+
+class TaxonomyOPAMigration extends Migration {
+ protected $key = 'code';
+
+ // Custom mappings as provided by modules implementing hook_taxonomy_opa_migrate_mapping_info();
+ protected $customMappings = array();
+
+ // Internal store of newly created or updated terms objects
+ protected $processedTerms = array();
+
+ public function __construct($arguments) {
+ parent::__construct($arguments);
+
+ $this->description = t('Import categories into Taxonomy terms.');
+
+ // Create a map object for tracking the relationships between source rows.
+ // If we want to enforce unique emission numbers also outside of this migration,
+ // we will use the MigrateUniqueEMNumberSQLMap class instead of the default
+ // MigrateSQLMap class
+ $mapClass = 'MigrateSQLMap';
+ if (isset($arguments['uniqueEmNumber']) && $arguments['uniqueEmNumber']) {
+ $mapClass = 'MigrateUniqueEMNumberSQLMap';
+ }
+
+ $this->map = new $mapClass(
+ $this->machineName,
+ array(
+ $this->key => array(
+ 'type' => 'varchar',
+ 'length' => 255,
+ ),
+ ),
+ MigrateDestinationTerm::getKeySchema()
+ );
+
+ // Destination
+ //$needsUpdate = isset($arguments['needsUpdate']) ? $arguments['needsUpdate'] : FALSE;
+ $this->destination = new MigrateDestinationTerm($arguments['bundle'] /*TODO*/);
+
+ // Get an OPA connection
+ $opa = arte_opa_get_api();
+
+ // Use debug configuration for OPA?
+ if (isset($arguments['debug']) && $arguments['debug']) {
+ $opa->setDebug();
+ }
+
+ // Combine with Rabbit
+ if (isset($arguments['rabbitmq']) && $arguments['rabbitmq']) {
+ // Provide default languages
+ $arguments['languages'] = arte_opa_languages();
+
+ $this->source = new MigrateSourceRabbitMQOPA(
+ array(
+ 'host' => variable_get('scald_opa_migrate_amqp_host', 'localhost'),
+ 'port' => variable_get('scald_opa_migrate_amqp_port', 5672),
+ 'user' => variable_get('scald_opa_migrate_amqp_user', 'guest'),
+ 'pass' => variable_get('scald_opa_migrate_amqp_pass', 'guest'),
+ 'vhost' => variable_get('scald_opa_migrate_amqp_vhost', '/'),
+ ),
+ variable_get('scald_opa_migrate_amqp_queue', 'category'),
+ $opa,
+ $arguments['query'],
+ $arguments['languages']
+ );
+ }
+ // Query OPA for the list
+ else {
+ // Source: OPA with parameters
+ $query = isset($arguments['query']) ? $arguments['query'] : array();
+ $this->source = new MigrateSourceTaxonomyOPA($opa, $query);
+ }
+
+ // Field mappings
+
+ // The language of the fields.
+ $this->addFieldMapping('language', 'language');
+
+ $this->addFieldMapping('name', 'label');
+ $this->addFieldMapping('weight', 'order');
+
+ // Grab custom mappings provided by other modules.
+ $this->customMappings = module_invoke_all('taxonomy_opa_migrate_mapping_info');
+ }
+
+ public function prepareRow($row) {
+ if (parent::prepareRow($row) === FALSE) {
+ return FALSE;
+ }
+ }
+
+ /**
+ * Called immediately after the complete destination object is saved.
+ */
+ public function complete($entity, $row) {
+ // Find out if this was a new taxonomy term or the updated of an existing one
+ $hook = isset($entity->original) ? 'update' : 'insert';
+
+ // Logging
+ $msg = 'Taxonomy term %verb: %no_emiarte %kind, %title.';
+ $vars = array(
+ '%verb' => isset($entity->original) ? 'updated' : 'created',
+ '%no_emiarte' => $entity->base_id,
+ '%kind' => $row->kind,
+ '%title' => $entity->title,
+ );
+ $link = l('Taxonomy term ' . $entity->tid, 'taxonomy/term/' . $entity->tid);
+ watchdog('taxonomy_opa_migrate', $msg, $vars, WATCHDOG_DEBUG, $link);
+
+ // Store processed atoms
+ $this->processedTerms[] = $entity;
+
+ // Notify that we have a new or updated scald atom.
+ module_invoke_all('taxonomy_opa_migrate_' . $hook, $entity);
+ }
+
+ /**
+ * Called when the import has finished
+ */
+ public function postImport() {
+ // Notify that the import has finished
+ module_invoke_all('taxonomy_opa_migrate_complete', $this->processedTerms);
+ }
+
+ /**
+ * Override of the prepareKey method.
+ *
+ * We use the code and the kind to identify unique taxonomy terms
+ * in Drupal.
+ *
+ * Unique key: code_kind
+ * Examples: MUA_INTERNET
+ *
+ * @param array $source_key
+ * @param object $row
+ *
+ * @return array
+ */
+ public function prepareKey($source_key, $row) {
+ $key = array();
+ foreach ($source_key as $field_name => $field_schema) {
+
+ // Start our unique key with the code
+ $val = $row->code;
+
+ // Append the 'kind' (e.g. SHOW, BONUS, etc.)
+ $val .= '_' . $row->kind;
+
+ $key[$field_name] = $val;
+ $row->{$field_name} = $val;
+ }
+ return $key;
+ }
+
+ /**
+ * Auxiliary function to extract the code from the key
+ *
+ * @param [type] $key [description]
+ * @return [type] [description]
+ */
+ public static function extractCodeFromKey($key) {
+ $parts = explode('_', $key);
+ if (count($parts) == 2) {
+ return $parts[0];
+ }
+ return FALSE;
+ }
+
+ /**
+ * Auxiliary function to extract the kind from the key.
+ * @param [type] $key [description]
+ * @return [type] [description]
+ */
+ public static function extractKindFromKey($key) {
+ $parts = explode('_', $key);
+ if (count($parts) == 2) {
+ return $parts[1];
+ }
+ return FALSE;
+ }
+}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
new file mode 100644
index 0000000..5dc7391
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
@@ -0,0 +1,143 @@
+<?php
+/**
+ * @file
+ *
+ * Provides drush extensions for Taxonomy OPA
+ */
+
+/**
+ * Implements hook_drush_command().
+ */
+function taxonomy_opa_migrate_drush_command() {
+ $items = array();
+
+ $items['taxonomy-opa-sync'] = array(
+ 'description' => 'Import categories from OPA into Taxonomy terms',
+ 'aliases' => array('tos'),
+ 'options' => array(
+ 'rabbitmq' => 'If this option is set, use the Rabbit Queue for getting a list of items to query from OPA',
+ 'platform' => 'The platform, e.g. ALW or AC. This is an obligatory option',
+ 'query' => 'E.g. "kind=INTERNET". Should have the format of URL GET parameters, and it might overwrite previously set parameters such as platform, channel or language.',
+ 'limit' => 'Limit on the length of each migration process, expressed in seconds or number of items',
+ //'update' => 'Also overwrite previously migrated items, if their last change date is the same. Can be used to force-input data from OPA in case of issues',
+ 'bundle' => 'The bundle of taxonomy',
+ 'feedback' => 'Provides feedback during migration, see migration feedback option for more information',
+ 'debug-opa' => 'Output debugging information about the OPA requests issued during synchronization',
+ //'uniqueEmNumber' => 'Enforce unique emission numbers, also outside the current migration. Can be used to update atoms that were previously migrated by a different migration class. Uniqueness is then defined using the emission number. Warning: do not use this if you might have duplicate emission numbers in the same system.',
+ ),
+ 'examples' => array(
+ 'tos --platform=ALW --feedback="10 seconds"' => 'Imports 10 videos (german) from the ALW platform having feedback every 10 seconds',
+ ),
+ );
+
+ return $items;
+}
+
+/**
+ * Command callback: run Taxonomy OPA synchronization
+ */
+function drush_taxonomy_opa_migrate_taxonomy_opa_sync($args = NULL) {
+ // Fetch the platform - this option is obligatory and will fail if not provided
+ $platform = drush_get_option('platform', FALSE);
+ if (!$platform || $platform == 1) {
+ drush_log(dt('Missing or empty parameter --platform'), 'error');
+ return;
+ }
+
+ $bundle = drush_get_option('bundle', FALSE);
+ if (!$bundle || $bundle == 1) {
+ drush_log(dt('Missing or empty parameter --bundle'), 'error');
+ return;
+ }
+
+ $arguments = array(
+ //'needsUpdate' => drush_get_option('update', FALSE),
+ 'debug' => drush_get_option('debug-opa', FALSE),
+ //'uniqueEmNumber' => drush_get_option('uniqueEmNumber', FALSE),
+ );
+
+ if (drush_get_option('rabbitmq', FALSE)) {
+ // Set up rabbitmq connection parameters
+ $arguments['rabbitmq'] = TRUE;
+
+ // Set the taxonomy bundle
+ $arguments['bundle'] = $bundle;
+
+ // Pass on more query parameters for filtering the Queue
+ $arguments['query'] = array();
+ $arguments['query']['platform'] = drush_get_option('platform');
+ $arguments['query']['kind'] = 'INTERNET';
+ }
+ else {
+ // Default query arguments
+ $arguments['query'] = array(
+ 'platform' => drush_get_option('platform'),
+ 'kind' => 'INTERNET',
+ );
+
+ // Set the taxonomy bundle
+ $arguments['bundle'] = $bundle;
+
+ // Additional query parameters
+ if ($query_additional = drush_get_option('query', FALSE)) {
+ $parts = explode('&', $query_additional);
+ if (!empty($parts)) {
+ foreach ($parts as $part) {
+ $elements = explode('=', $part);
+ if (count($elements) == 2) {
+ $arguments['query'][$elements[0]] = $elements[1];
+ }
+ }
+ }
+ }
+ }
+
+ // Limit
+ if ($limit = drush_get_option('limit')) {
+ $options['limit']['value'] = $limit;
+ $options['limit']['unit'] = 'items';
+ }
+
+ // Feedback
+ $feedback = drush_get_option('feedback');
+ if ($feedback) {
+ $parts = explode(' ', $feedback);
+ $options['feedback']['value'] = $parts[0];
+ $options['feedback']['unit'] = $parts[1];
+ if ($options['feedback']['unit'] != 'seconds' &&
+ $options['feedback']['unit'] != 'second' &&
+ $options['feedback']['unit'] != 'items' &&
+ $options['feedback']['unit'] != 'item') {
+ drush_set_error(NULL, dt("Invalid feedback frequency unit '!unit'",
+ array('!unit' => $options['feedback']['unit'])));
+ return;
+ }
+ }
+
+ // Run the migration
+ _drush_taxonomy_opa_migrate_run_migrate($arguments, $options);
+}
+
+/**
+ *
+ */
+function _drush_taxonomy_opa_migrate_run_migrate($arguments = array(), $options = array()) {
+ $machine_name = 'TaxonomyOPA';
+ Migration::registerMigration(
+ 'TaxonomyOPAMigration',
+ $machine_name,
+ $arguments + array('group_name' => 'opa')
+ );
+
+ $migration = Migration::getInstance($machine_name);
+
+ // We will update nodes as we go along.
+ $migration->prepareUpdate();
+
+ // Import!
+ $status = $migration->processImport($options);
+
+ // Reset all caches, if not recreating our migration class with different
+ // parameters won't make any difference
+ drupal_static_reset();
+}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
new file mode 100644
index 0000000..81a1e54
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
@@ -0,0 +1,19 @@
+name = Taxonomy OPA Migrate
+description = Provides Taxonomy Categories from OPA.
+package = ARTE Foundation
+core = 7.x
+dependencies[] = taxonomy
+dependencies[] = arte_opa
+
+; OPA Classes
+files[] = includes/opa_categories_reader.inc
+
+; Migrate source classes
+files[] = migrate/sources/taxonomy_opa.inc
+files[] = migrate/sources/taxonomy_rabbit_mq_opa.inc
+
+files[] = migrate/sources/unique_code_map.inc
+
+; Migrate classes
+files[] = migrate/taxonomy_opa_migrate.inc
+files[] = scald_opa_migrate.migrate.inc
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.install b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.install
new file mode 100644
index 0000000..b3d9bbc
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.install
@@ -0,0 +1 @@
+<?php
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
new file mode 100644
index 0000000..b3d9bbc
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
@@ -0,0 +1 @@
+<?php
From 5b06373398858f36aa749244a59ac792886e2bf2 Mon Sep 17 00:00:00 2001
From: lsantos-oq <luis.santos@openquest.pt>
Date: Thu, 19 May 2016 17:32:11 +0100
Subject: [PATCH 02/12] AFOUND-731 Add translations and corrections
---
modules/custom/taxonomy_opa_migrate/README.md | 50 +--
.../migrate/sources/rabbit_mq_opa_base.inc | 332 ------------------
.../migrate/sources/taxonomy_opa.inc | 6 +-
.../migrate/sources/taxonomy_rabbit_mq_opa.inc | 381 ---------------------
.../migrate/sources/unique_code_map.inc | 57 ---
.../migrate/taxonomy_opa_migrate.inc | 57 +--
.../taxonomy_opa_migrate.drush.inc | 60 ++--
.../taxonomy_opa_migrate/taxonomy_opa_migrate.info | 4 -
.../taxonomy_opa_migrate.install | 1 -
.../taxonomy_opa_migrate.module | 3 +
10 files changed, 66 insertions(+), 885 deletions(-)
delete mode 100644 modules/custom/taxonomy_opa_migrate/migrate/sources/rabbit_mq_opa_base.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_rabbit_mq_opa.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/migrate/sources/unique_code_map.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.install
diff --git a/modules/custom/taxonomy_opa_migrate/README.md b/modules/custom/taxonomy_opa_migrate/README.md
index beb2a10..41d4da0 100644
--- a/modules/custom/taxonomy_opa_migrate/README.md
+++ b/modules/custom/taxonomy_opa_migrate/README.md
@@ -3,6 +3,8 @@ Taxonomy OPA Migrate
Module providing support for migrating from OPA into `taxonomy terms`. Dynamic migrations use the migrate framework and Drush commands are supported to run migrations via the command-line.
+The Taxonomy that you select to put the migrate items need have the Translation mode: `Localize. Terms are common for all languages, but their name and description may be localized.`
+
### Configuring OPA
Follow configuration instructions in the `arte_opa` module to make sure you can connect to OPA.
@@ -17,54 +19,12 @@ See the Migrate class `taxonomy_opa.inc` for mapping information.
The default configuration is querying OPA directly with the given parameters (see "Drush commands" for the parameters to pass on).
-#### RabbitMQ
-
-Another method is to use RabbitMQ notifications to update taxonomy terms, given a notification that wraps the code. OPA is queried for every code.
-
-The advantage of this method is that it does not need to query OPA for full listings of videos, but instead will only add and update videos that are added to the queue, which is directly controlled by ARTE.
-
-Currently, RabbitMQ is supported and in production for ARTE Concert, using the rabbit_mq_opa Migrate source.
-
-To enable RabbitMQ, you can pass on the option `--rabbitmq` to the drush command, and you also need to configure the following variables (preferably, directly in `settings.php`)
-
- // OPA Rabbit settings (example from ARTE Concert production)
- $conf['scald_opa_migrate_amqp_host'] = 'ravel.arte.tv';
- $conf['scald_opa_migrate_amqp_port'] = '5672';
- $conf['scald_opa_migrate_amqp_user'] = 'concert';
- $conf['scald_opa_migrate_amqp_pass'] = '*******';
- $conf['scald_opa_migrate_amqp_vhost'] = '/mom_v2_prod';
- $conf['scald_opa_migrate_amqp_queue'] = 'opa_updates_for_concert';
-
### Drush commands
The module provides a custom Drush command for synchronization
- drush tos
-
-See `drush help sos` for examples. For example, to imports 10 videos (german) from the ALW platform
-
- drush tos --platform=ALW --bundle=genres
-
-Note that we map different language version onto *the same atom*, meaning that after running these two commands we'll have 10 opa atoms that contain both the German and the French versions (using Drupal's Entity Translation system).
-
-## RabbitMQ server
-
-This module also supports migrating from RabbitMQ. A message is sent with enough information to query OPA for the right video resources.
-
-For local testing, you can install RabbitMQ and fill up the queue with sample data.
-
-Start testing as follows
-
-`rabbitmq-server`
-Starts the server
-
-`php test/rabbitmq/populate_queue.php`
-Populates some test data in the queue (video). To change the test data, add files to `list.php`
-
-`rabbitmqadmin list queues`
-Check how many items are in the video queue
+ drush tos
-Now run the drush command to import in your site (see below).
+See `drush help tos` for examples. For example see examples
-Purges the queue (emptying it)
-`rabbitmqadmin purge queue name=opa`
+ drush tos --platform=ALW --bundle=genres
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/sources/rabbit_mq_opa_base.inc b/modules/custom/taxonomy_opa_migrate/migrate/sources/rabbit_mq_opa_base.inc
deleted file mode 100644
index 01c4b09..0000000
--- a/modules/custom/taxonomy_opa_migrate/migrate/sources/rabbit_mq_opa_base.inc
+++ /dev/null
@@ -1,332 +0,0 @@
-<?php
-
-// Use the AMQP namespace
-
-// This include path will only work if the scald_opa_migrate modules doesn't move within the profile.
-// @todo find a more elegant way to include the correct namespace
-require_once __DIR__ . '/../../../../../libraries/php-amqplib/vendor/autoload.php';
-use PhpAmqpLib\Connection\AMQPConnection;
-
-/**
- * @file
- * Define a MigrateSource for importing from APIOS using RabbitMQ
- */
-
-/**
- * Implementation of MigrateSource, to handle imports from Drupal connections.
- */
-class MigrateSourceRabbitMQOPABase extends MigrateSourceOPABase {
-
- /**
- * The AMQP connection.
- */
- protected $connection = NULL;
-
-
- /**
- * The AMQP channel object.
- */
- protected $channel = NULL;
-
- /**
- * The AMQP queue from which to fetch items.
- */
- protected $queue = '';
-
- /**
- * List of available source fields.
- *
- * @var array
- */
- protected $fields = array();
-
- /**
- * Supported languages to fetch videos for.
- *
- * @var array
- */
- protected $languages = array();
-
- /**
- * The current message number
- */
- protected $currentNumber = 0;
-
- /**
- * The total number of messages in the queue
- */
- protected $totalQueue = 0;
-
- /**
- * The OPA source
- */
- protected $opa = NULL;
-
-
- /**
- * Additional OPA source parameters
- */
- protected $params = array();
-
-
- /**
- * The mini-queue. Everytime we do a callback to OPA we might return a series
- * of videos in one go (e.g. all translations of a single video), these are then queued
- * up here and processed.
- */
- protected $miniQueue = array();
-
- /**
- * Return a string representing AMQP
- */
- public function __toString() {
- return "AMQP " . $this->queue;
- }
-
- /**
- * Connect to AMQP.
- */
- protected function connect() {
- if (!isset($this->channel)) {
- $this->connection = new AMQPConnection(
- $this->configuration['host'],
- $this->configuration['port'],
- $this->configuration['user'],
- $this->configuration['pass'],
- $this->configuration['vhost']
- );
- $this->channel = $this->connection->channel();
- $this->channel->queue_declare($this->queue, TRUE, TRUE);
- }
-
- if ($this->channel->active) {
- return TRUE;
- }
- else {
- throw new Exception(t("AMQP Channel is not active for queue !q", array('!q' => $this->queue)));
- return FALSE;
- }
- }
-
- /**
- * Disconnect from AMQP
- */
- protected function disconnect() {
- $this->channel->close();
- $this->connection->close();
- }
-
- /**
- * Simple initialization.
- */
- public function __construct(array $configuration, $queue, $opa, $params, $languages, $fields = array(), array $options = array()) {
- parent::__construct($options);
-
- // Load the PHP AMQP library
- libraries_load('php-amqplib');
-
- $this->queue = $queue;
- $this->configuration = $configuration;
- $this->languages = $languages;
- $this->fields = $fields;
- $this->params = $params;
-
- $this->connect();
-
- // Set the number of messages in the queue
- $this->totalQueue = $this->computeCount();
-
- // Save OPA source
- $this->opa = $opa;
- }
-
-
- function __destruct() {
- // Disconnect
- // @todo research why disconnecting doesn't work, it throws errors
- // $this->disconnect();
- }
-
- /**
- * Returns a list of fields available to be mapped from the source query.
- *
- * @return array
- * Keys: machine names of the fields (to be passed to addFieldMapping)
- * Values: Human-friendly descriptions of the fields.
- */
- public function fields() {
- return $this->fields;
- }
-
- /**
- * Return a count of all available source records.
- */
- public function computeCount() {
- if ($this->channel) {
- $ret = $this->channel->queue_declare($this->queue, TRUE, TRUE);
- if ($ret && isset($ret[1])) {
- return $ret[1];
- }
- else {
- return 0;
- }
- }
- else {
- // Queue isn't alive?
- return 0;
- }
- }
-
- /**
- * Implementation of MigrateSource::performRewind().
- *
- * @return void
- */
- public function performRewind() {
- // Nothing for now
- }
-
- /**
- * Implementation of MigrateSource::getNextRow().
- *
- * @return null|stdClass
- */
- public function getNextRow() {
- // First off-load any items from our mini queue.
- if (!empty($this->miniQueue)) {
- $return = array_shift($this->miniQueue);
- return $return;
- }
-
- // Make sure we are connected to Rabbit
- if (!$this->channel) {
- return NULL;
- }
-
- // Try fetching new messages from the queue now
- try {
- $msg = $this->channel->basic_get($this->queue);
- if ($msg) {
- $items = $this->fetchCategories($msg->body, $this->languages);
- $migration = Migration::currentMigration();
-
- // We received the videos correctly
- $this->currentNumber++;
-
- // Acknowledge message reception
- if (!variable_get('scald_opa_migrate_rabbitmq_disable_acknowledgement', FALSE)) {
- $this->channel->basic_ack($msg->delivery_info['delivery_tag']);
- $migration->displayMessage(
- format_string(
- 'Message !num/!total received from queue !queue',
- array(
- '!num' => $this->currentNumber,
- '!total' => $this->totalQueue,
- '!queue' => $this->queue,
- )
- ),
- 'status'
- );
- }
-
- // Add videos to our mini-queue so they can be processed
- $this->miniQueue = $items['videos'];
- return $this->getNextRow();
- }
- }
- // Catch OPA exceptions but continue loading off the queue
- catch (OPAException $e) {
- $migration = Migration::currentMigration();
-
- // Something went wrong in fetching videos from OPA, report
- $migration = Migration::currentMigration();
- $message = t('Loading of OPA JSON from the AQMP queue failed. Acknowledging message and moving on. Error message from OPA: !message. Original API call made to "!url". ', array('!message' => $e->getMessage(), '!url' => $e->getUrl()));
-
- $migration->getMap()->saveMessage(array($this->getCurrentKey()), $message, MigrationBase::MESSAGE_ERROR);
-
- // Acknowledge receival, even though we couldn't find anything in OPA
- $this->channel->basic_ack($msg->delivery_info['delivery_tag']);
-
- $migration->displayMessage(
- format_string(
- 'Message !num/!total received from queue !queue but not processed',
- array(
- '!num' => $this->currentNumber,
- '!total' => $this->totalQueue,
- '!queue' => $this->queue,
- )
- ),
- 'error'
- );
-
- return $this->getNextRow();
- }
- catch (Exception $e) {
- $migration = Migration::currentMigration();
- $message = t('Error getting item from the AMQP queue: %exp_message<br/><br/>%exp_trace', array('%exp_message' => $e->getMessage(), '%exp_trace' => $e->getTraceAsString()));
- $migration->getMap()->saveMessage(
- array($this->getCurrentKey()), $message, MigrationBase::MESSAGE_ERROR);
-
- return NULL;
- }
- }
-
- /**
- * Fetch an OPA reader for a given program ID.
- *
- * Even though this function is really simple, it gets separated into
- * its own function to make it easier to write test classes around OPA.
- */
- protected function getOPAReader($program_id) {
- $reader = new OPACategoriesReader($this->opa, array(
- 'programId' => $program_id
- ) + $this->params);
-
- return $reader;
- }
-
- /**
- * Implementation of Iterator::next() - subclasses of MigrateSource should
- * implement getNextRow() to retrieve the next valid source rocord to process.
- *
- * Adaptation to make sure we traverse every single line regardless
- * of whether we've imported this one before. Necessary since the same video can be added
- * twice.
- */
- public function next() {
- $this->currentKey = NULL;
- $this->currentRow = NULL;
- migrate_instrument_start(get_class($this) . ' getNextRow');
- while ($row = $this->getNextRow()) {
- migrate_instrument_stop(get_class($this) . ' getNextRow');
- // Populate the source key for this row
- $this->currentKey = $this->activeMigration->prepareKey(
- $this->activeMap->getSourceKey(), $row);
-
- // Pick up the existing map row, if any, unless getNextRow() did it.
- if (!$this->mapRowAdded) {
- $map_row = $this->activeMap->getRowBySource($this->currentKey);
- // Add map info to the row, if present
- if ($map_row) {
- foreach ($map_row as $field => $value) {
- $field = 'migrate_map_' . $field;
- $row->$field = $value;
- }
- }
- }
-
- if ($this->prepareRow($row) !== FALSE) {
- // Finally, we've got a keeper.
- $this->currentRow = $row;
- break;
- }
- else {
- $this->currentRow = NULL;
- }
- }
- migrate_instrument_stop(get_class($this) . ' getNextRow');
- if (!$this->currentRow) {
- $this->currentKey = NULL;
- }
- }
-}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc b/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc
index e07ff45..34be07e 100644
--- a/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc
+++ b/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc
@@ -47,7 +47,6 @@ class MigrateSourceTaxonomyOPA extends MigrateSource {
return count($this->reader);
}
catch (OPAException $e) {
- // @todo log somewhere?
return 0;
}
}
@@ -68,7 +67,7 @@ class MigrateSourceTaxonomyOPA extends MigrateSource {
public function getNextRow() {
migrate_instrument_start('MigrateSourceOPA::next');
- static $next = NULL;
+ $next = NULL;
if (!isset($next) && $this->reader->valid()) {
$next = (object)$this->reader->current();
@@ -86,8 +85,7 @@ class MigrateSourceTaxonomyOPA extends MigrateSource {
$next = (object)$this->reader->current();
// Keep looping on our results if $row and $next have the
- // same primary key, checking if there's more for that
- // program in the queue.
+ // same primary key
$same = ($row->code == $next->code && $row->kind == $next->kind && $row->language == $next->language);
}
else {
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_rabbit_mq_opa.inc b/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_rabbit_mq_opa.inc
deleted file mode 100644
index 5978084..0000000
--- a/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_rabbit_mq_opa.inc
+++ /dev/null
@@ -1,381 +0,0 @@
-<?php
-
-// Use the AMQP namespace
-
-// This include path will only work if the scald_opa_migrate modules doesn't move within the profile.
-// @todo find a more elegant way to include the correct namespace
-require_once __DIR__ . '/../../../../../libraries/php-amqplib/vendor/autoload.php';
-use PhpAmqpLib\Connection\AMQPConnection;
-
-/**
- * @file
- * Define a MigrateSource for importing from APIOS using RabbitMQ
- */
-
-/**
- * Implementation of MigrateSource, to handle imports from Drupal connections.
- */
-class MigrateSourceRabbitMQTaxonomyOPA extends MigrateSourceOPABase {
-
- /**
- * The AMQP connection.
- */
- protected $connection = NULL;
-
-
- /**
- * The AMQP channel object.
- */
- protected $channel = NULL;
-
- /**
- * The AMQP queue from which to fetch items.
- */
- protected $queue = '';
-
- /**
- * List of available source fields.
- *
- * @var array
- */
- protected $fields = array();
-
- /**
- * Supported languages to fetch videos for.
- *
- * @var array
- */
- protected $languages = array();
-
- /**
- * The current message number
- */
- protected $currentNumber = 0;
-
- /**
- * The total number of messages in the queue
- */
- protected $totalQueue = 0;
-
- /**
- * The OPA source
- */
- protected $opa = NULL;
-
-
- /**
- * Additional OPA source parameters
- */
- protected $params = array();
-
-
- /**
- * The mini-queue. Everytime we do a callback to OPA we might return a series
- * of videos in one go (e.g. all translations of a single video), these are then queued
- * up here and processed.
- */
- protected $miniQueue = array();
-
- /**
- * Return a string representing AMQP
- */
- public function __toString() {
- return "AMQP " . $this->queue;
- }
-
- /**
- * Connect to AMQP.
- */
- protected function connect() {
- if (!isset($this->channel)) {
- $this->connection = new AMQPConnection(
- $this->configuration['host'],
- $this->configuration['port'],
- $this->configuration['user'],
- $this->configuration['pass'],
- $this->configuration['vhost']
- );
- $this->channel = $this->connection->channel();
- $this->channel->queue_declare($this->queue, TRUE, TRUE);
- }
-
- if ($this->channel->active) {
- return TRUE;
- }
- else {
- throw new Exception(t("AMQP Channel is not active for queue !q", array('!q' => $this->queue)));
- return FALSE;
- }
- }
-
- /**
- * Disconnect from AMQP
- */
- protected function disconnect() {
- $this->channel->close();
- $this->connection->close();
- }
-
- /**
- * Simple initialization.
- */
- public function __construct(array $configuration, $queue, $opa, $params, $languages, $fields = array(), array $options = array()) {
- parent::__construct($options);
-
- // Load the PHP AMQP library
- libraries_load('php-amqplib');
-
- $this->queue = $queue;
- $this->configuration = $configuration;
- $this->languages = $languages;
- $this->fields = $fields;
- $this->params = $params;
-
- $this->connect();
-
- // Set the number of messages in the queue
- $this->totalQueue = $this->computeCount();
-
- // Save OPA source
- $this->opa = $opa;
- }
-
-
- function __destruct() {
- // Disconnect
- // @todo research why disconnecting doesn't work, it throws errors
- // $this->disconnect();
- }
-
- /**
- * Returns a list of fields available to be mapped from the source query.
- *
- * @return array
- * Keys: machine names of the fields (to be passed to addFieldMapping)
- * Values: Human-friendly descriptions of the fields.
- */
- public function fields() {
- return $this->fields;
- }
-
- /**
- * Return a count of all available source records.
- */
- public function computeCount() {
- if ($this->channel) {
- $ret = $this->channel->queue_declare($this->queue, TRUE, TRUE);
- if ($ret && isset($ret[1])) {
- return $ret[1];
- }
- else {
- return 0;
- }
- }
- else {
- // Queue isn't alive?
- return 0;
- }
- }
-
- /**
- * Implementation of MigrateSource::performRewind().
- *
- * @return void
- */
- public function performRewind() {
- // Nothing for now
- }
-
- /**
- * Implementation of MigrateSource::getNextRow().
- *
- * @return null|stdClass
- */
- public function getNextRow() {
- // First off-load any items from our mini queue.
- if (!empty($this->miniQueue)) {
- $return = array_shift($this->miniQueue);
- return $return;
- }
-
- // Make sure we are connected to Rabbit
- if (!$this->channel) {
- return NULL;
- }
-
- // Try fetching new messages from the queue now
- try {
- $msg = $this->channel->basic_get($this->queue);
- if ($msg) {
- $items = $this->fetchCategories($msg->body, $this->languages);
- $migration = Migration::currentMigration();
-
- // We received the videos correctly
- $this->currentNumber++;
-
- // Acknowledge message reception
- if (!variable_get('scald_opa_migrate_rabbitmq_disable_acknowledgement', FALSE)) {
- $this->channel->basic_ack($msg->delivery_info['delivery_tag']);
- $migration->displayMessage(
- format_string(
- 'Message !num/!total received from queue !queue',
- array(
- '!num' => $this->currentNumber,
- '!total' => $this->totalQueue,
- '!queue' => $this->queue,
- )
- ),
- 'status'
- );
- }
-
- // Add videos to our mini-queue so they can be processed
- $this->miniQueue = $items['videos'];
- return $this->getNextRow();
- }
- }
- // Catch OPA exceptions but continue loading off the queue
- catch (OPAException $e) {
- $migration = Migration::currentMigration();
-
- // Something went wrong in fetching videos from OPA, report
- $migration = Migration::currentMigration();
- $message = t('Loading of OPA JSON from the AQMP queue failed. Acknowledging message and moving on. Error message from OPA: !message. Original API call made to "!url". ', array('!message' => $e->getMessage(), '!url' => $e->getUrl()));
-
- $migration->getMap()->saveMessage(array($this->getCurrentKey()), $message, MigrationBase::MESSAGE_ERROR);
-
- // Acknowledge receival, even though we couldn't find anything in OPA
- $this->channel->basic_ack($msg->delivery_info['delivery_tag']);
-
- $migration->displayMessage(
- format_string(
- 'Message !num/!total received from queue !queue but not processed',
- array(
- '!num' => $this->currentNumber,
- '!total' => $this->totalQueue,
- '!queue' => $this->queue,
- )
- ),
- 'error'
- );
-
- return $this->getNextRow();
- }
- catch (Exception $e) {
- $migration = Migration::currentMigration();
- $message = t('Error getting item from the AMQP queue: %exp_message<br/><br/>%exp_trace', array('%exp_message' => $e->getMessage(), '%exp_trace' => $e->getTraceAsString()));
- $migration->getMap()->saveMessage(
- array($this->getCurrentKey()), $message, MigrationBase::MESSAGE_ERROR);
-
- return NULL;
- }
- }
-
- /**
- * Fetch an OPA reader for a given program ID.
- *
- * Even though this function is really simple, it gets separated into
- * its own function to make it easier to write test classes around OPA.
- */
- protected function getOPAReader($program_id) {
- $reader = new OPACategoriesReader($this->opa, array(
- 'programId' => $program_id
- ) + $this->params);
-
- return $reader;
- }
-
- /**
- * Fetch videos from a rabbit message.
- *
- * This will use the programId of the Rabbit message, and query OPA to fetch
- * informations about that program. Each program ID can be found both in
- * multiple languages and multiple platforms ; for each language, only find
- * the video that's best suited for use on the website.
- *
- * @param [type] $msg [description]
- * @param [type] $languages Array of supported language codes we should consider.
- * @return [type] [description]
- */
- protected function fetchCategories($msg, $languages) {
- $item = drupal_json_decode($msg);
-
- // Load up the right item now
- $reader = $this->getOPAReader($item['programId']);
-
- $categories = array(
- 'programId' => $item['programId'],
- 'categories' => array(),
- );
-
- foreach ($reader as $key => $value) {
- // Only process items in the supported languages
- if (in_array($value['language'], $languages)) {
- // Convert to an object
- $row = new stdClass();
- foreach ($value as $k => $v) {
- $row->$k = $v;
- }
-
- if (empty($categories['categories'][$row->language])) {
- $categories['categories'][$row->language] = $row;
- }
- else {
- // Check if this video is better suited for importing
- // than the other one in the same language returned by
- // OPA.
- $original_row = $categories['categories'][$row->language];
- $categories['categories'][$row->language] = $this->pickBestSuitedRecord($row, $original_row);
- }
- }
- }
-
- return $videos;
- }
-
-
- /**
- * Implementation of Iterator::next() - subclasses of MigrateSource should
- * implement getNextRow() to retrieve the next valid source rocord to process.
- *
- * Adaptation to make sure we traverse every single line regardless
- * of whether we've imported this one before. Necessary since the same video can be added
- * twice.
- */
- public function next() {
- $this->currentKey = NULL;
- $this->currentRow = NULL;
- migrate_instrument_start(get_class($this) . ' getNextRow');
- while ($row = $this->getNextRow()) {
- migrate_instrument_stop(get_class($this) . ' getNextRow');
- // Populate the source key for this row
- $this->currentKey = $this->activeMigration->prepareKey(
- $this->activeMap->getSourceKey(), $row);
-
- // Pick up the existing map row, if any, unless getNextRow() did it.
- if (!$this->mapRowAdded) {
- $map_row = $this->activeMap->getRowBySource($this->currentKey);
- // Add map info to the row, if present
- if ($map_row) {
- foreach ($map_row as $field => $value) {
- $field = 'migrate_map_' . $field;
- $row->$field = $value;
- }
- }
- }
-
- if ($this->prepareRow($row) !== FALSE) {
- // Finally, we've got a keeper.
- $this->currentRow = $row;
- break;
- }
- else {
- $this->currentRow = NULL;
- }
- }
- migrate_instrument_stop(get_class($this) . ' getNextRow');
- if (!$this->currentRow) {
- $this->currentKey = NULL;
- }
- }
-}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/sources/unique_code_map.inc b/modules/custom/taxonomy_opa_migrate/migrate/sources/unique_code_map.inc
deleted file mode 100644
index 473c941..0000000
--- a/modules/custom/taxonomy_opa_migrate/migrate/sources/unique_code_map.inc
+++ /dev/null
@@ -1,57 +0,0 @@
-<?php
-
-/**
- * @file
- *
- * Extends SQL map to load unique records if they already exist beyond this map.
- * Uniqueness is being calculated using the code of the category.
- *
- * This can be used to update data that was previously migrated using different
- * migrations or sources, to avoid that duplicates are being created.
- */
-
-class MigrateUniqueCodeSQLMap extends MigrateSQLMap {
-
- /**
- * Retrieve a row from the map table, given a source ID
- *
- * @param array $source_id
- */
- public function getRowBySource(array $source_id) {
- $source_id = array_shift($source_id);
-var_dump($source_id);exit;
- migrate_instrument_start('mapRowBySource');
- $query = $this->connection->select($this->mapTable, 'map')->fields('map');
- foreach ($this->sourceKeyMap as $key_name) {
- $query = $query->condition("map.$key_name", $source_id, '=');
- }
- $result = $query->execute();
- $res = $result->fetchAssoc();
-
- // This is where we change normal behavior:
- // if nothing can be found in the map, we check if we can find this using the
- // source ID. If we can find a nid, we pass this along and Migrate will adopt
- // this from now on and add this to the map.
- if (!$res && $em = ScaldOPAMigration::extractEMNumberFromKey($source_id)) {
- $sid = scald_search(array('base_id' => $em, 'type' => 'video'), FALSE, TRUE);
- if ($sid) {
- // We actually found a previously imported atom with that emission number, use that atom instead
- $res = array(
- 'sourceid1' => $source_id,
- 'destid1' => $sid,
- 'needs_update' => 1,
- 'rollback_action' => 0,
- 'last_imported' => 0,
- );
-
- // Log this action
- Migration::displayMessage(t('Used previously imported atom with sid !sid for emission number !em that was not present in mapping table.', array('!sid' => $sid, '!em' => $em)), 'warning');
- }
- }
-
- migrate_instrument_stop('mapRowBySource');
-
- // Return the result
- return $res;
- }
-}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
index 289bf4a..10888b6 100644
--- a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
+++ b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
@@ -11,6 +11,12 @@ class TaxonomyOPAMigration extends Migration {
// Custom mappings as provided by modules implementing hook_taxonomy_opa_migrate_mapping_info();
protected $customMappings = array();
+ // OPA connection
+ protected $opa = null;
+
+ // The translation query
+ protected $translationsQuery = array();
+
// Internal store of newly created or updated terms objects
protected $processedTerms = array();
@@ -40,42 +46,28 @@ class TaxonomyOPAMigration extends Migration {
);
// Destination
- //$needsUpdate = isset($arguments['needsUpdate']) ? $arguments['needsUpdate'] : FALSE;
- $this->destination = new MigrateDestinationTerm($arguments['bundle'] /*TODO*/);
+ $this->destination = new MigrateDestinationTerm($arguments['bundle']);
// Get an OPA connection
- $opa = arte_opa_get_api();
+ $this->opa = arte_opa_get_api();
// Use debug configuration for OPA?
if (isset($arguments['debug']) && $arguments['debug']) {
$opa->setDebug();
}
- // Combine with Rabbit
- if (isset($arguments['rabbitmq']) && $arguments['rabbitmq']) {
- // Provide default languages
- $arguments['languages'] = arte_opa_languages();
-
- $this->source = new MigrateSourceRabbitMQOPA(
- array(
- 'host' => variable_get('scald_opa_migrate_amqp_host', 'localhost'),
- 'port' => variable_get('scald_opa_migrate_amqp_port', 5672),
- 'user' => variable_get('scald_opa_migrate_amqp_user', 'guest'),
- 'pass' => variable_get('scald_opa_migrate_amqp_pass', 'guest'),
- 'vhost' => variable_get('scald_opa_migrate_amqp_vhost', '/'),
- ),
- variable_get('scald_opa_migrate_amqp_queue', 'category'),
- $opa,
- $arguments['query'],
- $arguments['languages']
- );
- }
- // Query OPA for the list
- else {
- // Source: OPA with parameters
- $query = isset($arguments['query']) ? $arguments['query'] : array();
- $this->source = new MigrateSourceTaxonomyOPA($opa, $query);
+ // Source: OPA with parameters
+ $query = isset($arguments['query']) ? $arguments['query'] : array();
+ $this->source = new MigrateSourceTaxonomyOPA($this->opa, $query);
+
+ // Build the translation query
+ $this->translationsQuery = isset($arguments['query']) ? $arguments['query'] : array();
+ $languages_enabled = arte_opa_languages();
+
+ if (in_array($arguments['query']['language'], $languages_enabled)) {
+ unset($languages_enabled[$arguments['query']['language']]);
}
+ $this->translationsQuery['language'] = implode(',', $languages_enabled);
// Field mappings
@@ -116,6 +108,17 @@ class TaxonomyOPAMigration extends Migration {
// Store processed atoms
$this->processedTerms[] = $entity;
+ // Get the translations
+ $query = $this->translationsQuery;
+ $query['code'] = $this->extractCodeFromKey($row->code);
+ $i18n_category = new MigrateSourceTaxonomyOPA($this->opa, $query);
+
+ while ($i18n_row = $i18n_category->getNextRow()) {
+ var_dump($i18n_row);
+ $report = array();
+ _locale_import_one_string_db($report, $i18n_row->language, 'term:'. $entity->tid .':name', $row->label, $i18n_row->label, 'Taxonomy', '', LOCALE_IMPORT_OVERWRITE);
+ }
+
// Notify that we have a new or updated scald atom.
module_invoke_all('taxonomy_opa_migrate_' . $hook, $entity);
}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
index 5dc7391..29385ad 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
@@ -15,18 +15,15 @@ function taxonomy_opa_migrate_drush_command() {
'description' => 'Import categories from OPA into Taxonomy terms',
'aliases' => array('tos'),
'options' => array(
- 'rabbitmq' => 'If this option is set, use the Rabbit Queue for getting a list of items to query from OPA',
'platform' => 'The platform, e.g. ALW or AC. This is an obligatory option',
- 'query' => 'E.g. "kind=INTERNET". Should have the format of URL GET parameters, and it might overwrite previously set parameters such as platform, channel or language.',
+ 'query' => 'E.g. "sort=language". Should have the format of URL GET parameters, and it might overwrite previously set parameters such as platform, code or language.',
'limit' => 'Limit on the length of each migration process, expressed in seconds or number of items',
- //'update' => 'Also overwrite previously migrated items, if their last change date is the same. Can be used to force-input data from OPA in case of issues',
'bundle' => 'The bundle of taxonomy',
'feedback' => 'Provides feedback during migration, see migration feedback option for more information',
'debug-opa' => 'Output debugging information about the OPA requests issued during synchronization',
- //'uniqueEmNumber' => 'Enforce unique emission numbers, also outside the current migration. Can be used to update atoms that were previously migrated by a different migration class. Uniqueness is then defined using the emission number. Warning: do not use this if you might have duplicate emission numbers in the same system.',
),
'examples' => array(
- 'tos --platform=ALW --feedback="10 seconds"' => 'Imports 10 videos (german) from the ALW platform having feedback every 10 seconds',
+ 'tos --platform=ALW --bundle=genres --feedback="10 seconds"' => 'Imports the categories to the genres Taxonomy from the ALW platform having feedback every 10 seconds',
),
);
@@ -56,37 +53,25 @@ function drush_taxonomy_opa_migrate_taxonomy_opa_sync($args = NULL) {
//'uniqueEmNumber' => drush_get_option('uniqueEmNumber', FALSE),
);
- if (drush_get_option('rabbitmq', FALSE)) {
- // Set up rabbitmq connection parameters
- $arguments['rabbitmq'] = TRUE;
- // Set the taxonomy bundle
- $arguments['bundle'] = $bundle;
+ // Default query arguments
+ $arguments['query'] = array(
+ 'platform' => drush_get_option('platform'),
+ 'kind' => 'INTERNET',
+ 'language' => variable_get('taxonomy_opa_migrate_base_language', 'en'),
+ );
- // Pass on more query parameters for filtering the Queue
- $arguments['query'] = array();
- $arguments['query']['platform'] = drush_get_option('platform');
- $arguments['query']['kind'] = 'INTERNET';
- }
- else {
- // Default query arguments
- $arguments['query'] = array(
- 'platform' => drush_get_option('platform'),
- 'kind' => 'INTERNET',
- );
-
- // Set the taxonomy bundle
- $arguments['bundle'] = $bundle;
-
- // Additional query parameters
- if ($query_additional = drush_get_option('query', FALSE)) {
- $parts = explode('&', $query_additional);
- if (!empty($parts)) {
- foreach ($parts as $part) {
- $elements = explode('=', $part);
- if (count($elements) == 2) {
- $arguments['query'][$elements[0]] = $elements[1];
- }
+ // Set the taxonomy bundle
+ $arguments['bundle'] = $bundle;
+
+ // Additional query parameters
+ if ($query_additional = drush_get_option('query', FALSE)) {
+ $parts = explode('&', $query_additional);
+ if (!empty($parts)) {
+ foreach ($parts as $part) {
+ $elements = explode('=', $part);
+ if (count($elements) == 2) {
+ $arguments['query'][$elements[0]] = $elements[1];
}
}
}
@@ -140,4 +125,11 @@ function _drush_taxonomy_opa_migrate_run_migrate($arguments = array(), $options
// Reset all caches, if not recreating our migration class with different
// parameters won't make any difference
drupal_static_reset();
+
+ // Clear cache and force refresh of JavaScript translations.
+ _locale_invalidate_js($langcode);
+ cache_clear_all('locale:', 'cache', TRUE);
+
+ // Rebuild the menu, strings may have changed.
+ menu_rebuild();
}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
index 81a1e54..205a35b 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
@@ -10,10 +10,6 @@ files[] = includes/opa_categories_reader.inc
; Migrate source classes
files[] = migrate/sources/taxonomy_opa.inc
-files[] = migrate/sources/taxonomy_rabbit_mq_opa.inc
-
-files[] = migrate/sources/unique_code_map.inc
; Migrate classes
files[] = migrate/taxonomy_opa_migrate.inc
-files[] = scald_opa_migrate.migrate.inc
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.install b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.install
deleted file mode 100644
index b3d9bbc..0000000
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.install
+++ /dev/null
@@ -1 +0,0 @@
-<?php
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
index b3d9bbc..5759387 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
@@ -1 +1,4 @@
<?php
+/**
+ * Taxonomy Opa Migrate Module
+ */
From 1d9e56991cfe88baef972fe817a73aae4d2b7774 Mon Sep 17 00:00:00 2001
From: lsantos-oq <luis.santos@openquest.pt>
Date: Tue, 24 May 2016 11:52:34 +0100
Subject: [PATCH 03/12] AFOUND-731 Remove debug
---
modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc | 1 -
1 file changed, 1 deletion(-)
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
index 10888b6..5143138 100644
--- a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
+++ b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
@@ -114,7 +114,6 @@ class TaxonomyOPAMigration extends Migration {
$i18n_category = new MigrateSourceTaxonomyOPA($this->opa, $query);
while ($i18n_row = $i18n_category->getNextRow()) {
- var_dump($i18n_row);
$report = array();
_locale_import_one_string_db($report, $i18n_row->language, 'term:'. $entity->tid .':name', $row->label, $i18n_row->label, 'Taxonomy', '', LOCALE_IMPORT_OVERWRITE);
}
From 8068b40202e8c9d237747c1686a783fcb2c4ede3 Mon Sep 17 00:00:00 2001
From: lsantos-oq <luis.santos@openquest.pt>
Date: Tue, 24 May 2016 14:19:01 +0100
Subject: [PATCH 04/12] AFOUND-731 Create feature to export the taxonomy
---
modules/custom/taxonomy_opa_migrate/README.md | 2 +-
.../migrate/taxonomy_opa_migrate.inc | 60 +---------------------
.../taxonomy_opa_migrate.drush.inc | 11 +---
.../taxonomy_opa_migrate.features.field_base.inc | 39 ++++++++++++++
...axonomy_opa_migrate.features.field_instance.inc | 58 +++++++++++++++++++++
.../taxonomy_opa_migrate.features.inc | 14 +++++
.../taxonomy_opa_migrate.features.taxonomy.inc | 38 ++++++++++++++
.../taxonomy_opa_migrate/taxonomy_opa_migrate.info | 16 ++++--
.../taxonomy_opa_migrate.module | 5 +-
9 files changed, 169 insertions(+), 74 deletions(-)
create mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.inc
create mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.taxonomy.inc
diff --git a/modules/custom/taxonomy_opa_migrate/README.md b/modules/custom/taxonomy_opa_migrate/README.md
index 41d4da0..58d67b0 100644
--- a/modules/custom/taxonomy_opa_migrate/README.md
+++ b/modules/custom/taxonomy_opa_migrate/README.md
@@ -27,4 +27,4 @@ The module provides a custom Drush command for synchronization
See `drush help tos` for examples. For example see examples
- drush tos --platform=ALW --bundle=genres
+ drush tos --platform=ALW
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
index 5143138..97cb53d 100644
--- a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
+++ b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
@@ -76,6 +76,7 @@ class TaxonomyOPAMigration extends Migration {
$this->addFieldMapping('name', 'label');
$this->addFieldMapping('weight', 'order');
+ $this->addFieldMapping('field_opa_code', 'code');
// Grab custom mappings provided by other modules.
$this->customMappings = module_invoke_all('taxonomy_opa_migrate_mapping_info');
@@ -110,7 +111,7 @@ class TaxonomyOPAMigration extends Migration {
// Get the translations
$query = $this->translationsQuery;
- $query['code'] = $this->extractCodeFromKey($row->code);
+ $query['code'] = $row->code;
$i18n_category = new MigrateSourceTaxonomyOPA($this->opa, $query);
while ($i18n_row = $i18n_category->getNextRow()) {
@@ -129,61 +130,4 @@ class TaxonomyOPAMigration extends Migration {
// Notify that the import has finished
module_invoke_all('taxonomy_opa_migrate_complete', $this->processedTerms);
}
-
- /**
- * Override of the prepareKey method.
- *
- * We use the code and the kind to identify unique taxonomy terms
- * in Drupal.
- *
- * Unique key: code_kind
- * Examples: MUA_INTERNET
- *
- * @param array $source_key
- * @param object $row
- *
- * @return array
- */
- public function prepareKey($source_key, $row) {
- $key = array();
- foreach ($source_key as $field_name => $field_schema) {
-
- // Start our unique key with the code
- $val = $row->code;
-
- // Append the 'kind' (e.g. SHOW, BONUS, etc.)
- $val .= '_' . $row->kind;
-
- $key[$field_name] = $val;
- $row->{$field_name} = $val;
- }
- return $key;
- }
-
- /**
- * Auxiliary function to extract the code from the key
- *
- * @param [type] $key [description]
- * @return [type] [description]
- */
- public static function extractCodeFromKey($key) {
- $parts = explode('_', $key);
- if (count($parts) == 2) {
- return $parts[0];
- }
- return FALSE;
- }
-
- /**
- * Auxiliary function to extract the kind from the key.
- * @param [type] $key [description]
- * @return [type] [description]
- */
- public static function extractKindFromKey($key) {
- $parts = explode('_', $key);
- if (count($parts) == 2) {
- return $parts[1];
- }
- return FALSE;
- }
}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
index 29385ad..19b083d 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
@@ -18,12 +18,11 @@ function taxonomy_opa_migrate_drush_command() {
'platform' => 'The platform, e.g. ALW or AC. This is an obligatory option',
'query' => 'E.g. "sort=language". Should have the format of URL GET parameters, and it might overwrite previously set parameters such as platform, code or language.',
'limit' => 'Limit on the length of each migration process, expressed in seconds or number of items',
- 'bundle' => 'The bundle of taxonomy',
'feedback' => 'Provides feedback during migration, see migration feedback option for more information',
'debug-opa' => 'Output debugging information about the OPA requests issued during synchronization',
),
'examples' => array(
- 'tos --platform=ALW --bundle=genres --feedback="10 seconds"' => 'Imports the categories to the genres Taxonomy from the ALW platform having feedback every 10 seconds',
+ 'tos --platform=ALW --feedback="10 seconds"' => 'Imports the categories to the Foundation genres Taxonomy from the ALW platform having feedback every 10 seconds',
),
);
@@ -41,12 +40,6 @@ function drush_taxonomy_opa_migrate_taxonomy_opa_sync($args = NULL) {
return;
}
- $bundle = drush_get_option('bundle', FALSE);
- if (!$bundle || $bundle == 1) {
- drush_log(dt('Missing or empty parameter --bundle'), 'error');
- return;
- }
-
$arguments = array(
//'needsUpdate' => drush_get_option('update', FALSE),
'debug' => drush_get_option('debug-opa', FALSE),
@@ -62,7 +55,7 @@ function drush_taxonomy_opa_migrate_taxonomy_opa_sync($args = NULL) {
);
// Set the taxonomy bundle
- $arguments['bundle'] = $bundle;
+ $arguments['bundle'] = 'foundation_opa_genres';
// Additional query parameters
if ($query_additional = drush_get_option('query', FALSE)) {
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc
new file mode 100644
index 0000000..e75ff3b
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc
@@ -0,0 +1,39 @@
+<?php
+/**
+ * @file
+ * taxonomy_opa_migrate.features.field_base.inc
+ */
+
+/**
+ * Implements hook_field_default_field_bases().
+ */
+function taxonomy_opa_migrate_field_default_field_bases() {
+ $field_bases = array();
+
+ // Exported field_base: 'field_opa_code'.
+ $field_bases['field_opa_code'] = array(
+ 'active' => 1,
+ 'cardinality' => 1,
+ 'deleted' => 0,
+ 'entity_types' => array(),
+ 'field_name' => 'field_opa_code',
+ 'field_permissions' => array(
+ 'type' => 0,
+ ),
+ 'indexes' => array(
+ 'format' => array(
+ 0 => 'format',
+ ),
+ ),
+ 'locked' => 0,
+ 'module' => 'text',
+ 'settings' => array(
+ 'entity_translation_sync' => FALSE,
+ 'max_length' => 60,
+ ),
+ 'translatable' => 0,
+ 'type' => 'text',
+ );
+
+ return $field_bases;
+}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc
new file mode 100644
index 0000000..266b97c
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc
@@ -0,0 +1,58 @@
+<?php
+/**
+ * @file
+ * taxonomy_opa_migrate.features.field_instance.inc
+ */
+
+/**
+ * Implements hook_field_default_field_instances().
+ */
+function taxonomy_opa_migrate_field_default_field_instances() {
+ $field_instances = array();
+
+ // Exported field_instance:
+ // 'taxonomy_term-foundation_opa_genres-field_opa_code'.
+ $field_instances['taxonomy_term-foundation_opa_genres-field_opa_code'] = array(
+ 'bundle' => 'foundation_opa_genres',
+ 'default_value' => NULL,
+ 'deleted' => 0,
+ 'description' => '',
+ 'display' => array(
+ 'default' => array(
+ 'label' => 'above',
+ 'module' => 'text',
+ 'settings' => array(),
+ 'type' => 'text_default',
+ 'weight' => 0,
+ ),
+ ),
+ 'entity_type' => 'taxonomy_term',
+ 'field_name' => 'field_opa_code',
+ 'label' => 'Code',
+ 'required' => 1,
+ 'settings' => array(
+ 'context' => 'title',
+ 'context_default' => 'sdl_editor_representation',
+ 'dnd_enabled' => 0,
+ 'entity_translation_sync' => FALSE,
+ 'mee_enabled' => 0,
+ 'text_processing' => 0,
+ 'user_register_form' => FALSE,
+ ),
+ 'widget' => array(
+ 'active' => 1,
+ 'module' => 'text',
+ 'settings' => array(
+ 'size' => 60,
+ ),
+ 'type' => 'text_textfield',
+ 'weight' => 41,
+ ),
+ );
+
+ // Translatables
+ // Included for use with string extractors like potx.
+ t('Code');
+
+ return $field_instances;
+}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.inc
new file mode 100644
index 0000000..9d44dae
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.inc
@@ -0,0 +1,14 @@
+<?php
+/**
+ * @file
+ * taxonomy_opa_migrate.features.inc
+ */
+
+/**
+ * Implements hook_ctools_plugin_api().
+ */
+function taxonomy_opa_migrate_ctools_plugin_api($module = NULL, $api = NULL) {
+ if ($module == "strongarm" && $api == "strongarm") {
+ return array("version" => "1");
+ }
+}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.taxonomy.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.taxonomy.inc
new file mode 100644
index 0000000..3b98ab1
--- /dev/null
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.taxonomy.inc
@@ -0,0 +1,38 @@
+<?php
+/**
+ * @file
+ * taxonomy_opa_migrate.features.taxonomy.inc
+ */
+
+/**
+ * Implements hook_taxonomy_default_vocabularies().
+ */
+function taxonomy_opa_migrate_taxonomy_default_vocabularies() {
+ return array(
+ 'foundation_opa_genres' => array(
+ 'name' => 'Foundation Genres from OPA',
+ 'machine_name' => 'foundation_opa_genres',
+ 'description' => '',
+ 'hierarchy' => 0,
+ 'module' => 'taxonomy',
+ 'weight' => 0,
+ 'language' => 'und',
+ 'i18n_mode' => 1,
+ 'rdf_mapping' => array(
+ 'rdftype' => array(
+ 0 => 'skos:ConceptScheme',
+ ),
+ 'name' => array(
+ 'predicates' => array(
+ 0 => 'dc:title',
+ ),
+ ),
+ 'description' => array(
+ 'predicates' => array(
+ 0 => 'rdfs:comment',
+ ),
+ ),
+ ),
+ ),
+ );
+}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
index 205a35b..d9c9ae9 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
@@ -1,15 +1,21 @@
name = Taxonomy OPA Migrate
description = Provides Taxonomy Categories from OPA.
-package = ARTE Foundation
core = 7.x
-dependencies[] = taxonomy
+package = ARTE Foundation
dependencies[] = arte_opa
-
+dependencies[] = ctools
+dependencies[] = features
+dependencies[] = strongarm
+dependencies[] = taxonomy
+dependencies[] = text
+features[ctools][] = strongarm:strongarm:1
+features[features_api][] = api:2
+features[field_base][] = field_opa_code
+features[field_instance][] = taxonomy_term-foundation_opa_genres-field_opa_code
+features[taxonomy][] = foundation_opa_genres
; OPA Classes
files[] = includes/opa_categories_reader.inc
-
; Migrate source classes
files[] = migrate/sources/taxonomy_opa.inc
-
; Migrate classes
files[] = migrate/taxonomy_opa_migrate.inc
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
index 5759387..a169818 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
@@ -1,4 +1,7 @@
<?php
/**
- * Taxonomy Opa Migrate Module
+ * @file
+ * Code for the taxonomy_opa_migrate feature.
*/
+
+include_once 'taxonomy_opa_migrate.features.inc';
From 62066fe9d4bccdf1aee32ee117f53ae71e19d5ff Mon Sep 17 00:00:00 2001
From: lsantos-oq <luis.santos@openquest.pt>
Date: Fri, 27 May 2016 17:34:28 +0100
Subject: [PATCH 05/12] AFOUND-731 Create a new field for reference terms in
the foundation OPA Genres
---
.../migrate/taxonomy_opa_migrate.inc | 4 ---
.../taxonomy_opa_migrate.features.field_base.inc | 31 ++++++++++++++++++++
...axonomy_opa_migrate.features.field_instance.inc | 33 ++++++++++++++++++++++
.../taxonomy_opa_migrate/taxonomy_opa_migrate.info | 4 +++
.../taxonomy_opa_migrate.module | 17 +++++++++++
5 files changed, 85 insertions(+), 4 deletions(-)
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
index 97cb53d..80667d7 100644
--- a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
+++ b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
@@ -70,10 +70,6 @@ class TaxonomyOPAMigration extends Migration {
$this->translationsQuery['language'] = implode(',', $languages_enabled);
// Field mappings
-
- // The language of the fields.
- $this->addFieldMapping('language', 'language');
-
$this->addFieldMapping('name', 'label');
$this->addFieldMapping('weight', 'order');
$this->addFieldMapping('field_opa_code', 'code');
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc
index e75ff3b..1b16150 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc
@@ -35,5 +35,36 @@ function taxonomy_opa_migrate_field_default_field_bases() {
'type' => 'text',
);
+ // Exported field_base: 'field_video_opa_genre'.
+ $field_bases['field_video_opa_genre'] = array(
+ 'active' => 1,
+ 'cardinality' => -1,
+ 'deleted' => 0,
+ 'entity_types' => array(),
+ 'field_name' => 'field_video_opa_genre',
+ 'field_permissions' => array(
+ 'type' => 0,
+ ),
+ 'indexes' => array(
+ 'tid' => array(
+ 0 => 'tid',
+ ),
+ ),
+ 'locked' => 0,
+ 'module' => 'taxonomy',
+ 'settings' => array(
+ 'allowed_values' => array(
+ 0 => array(
+ 'vocabulary' => 'foundation_opa_genres',
+ 'parent' => 0,
+ ),
+ ),
+ 'entity_translation_sync' => FALSE,
+ 'options_list_callback' => 'i18n_taxonomy_allowed_values',
+ ),
+ 'translatable' => 0,
+ 'type' => 'taxonomy_term_reference',
+ );
+
return $field_bases;
}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc
index 266b97c..1f96e01 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc
@@ -10,6 +10,38 @@
function taxonomy_opa_migrate_field_default_field_instances() {
$field_instances = array();
+ // Exported field_instance: 'scald_atom-video-field_video_opa_genre'.
+ $field_instances['scald_atom-video-field_video_opa_genre'] = array(
+ 'bundle' => 'video',
+ 'default_value' => NULL,
+ 'deleted' => 0,
+ 'description' => '',
+ 'display' => array(
+ 'default' => array(
+ 'label' => 'above',
+ 'module' => 'i18n_taxonomy',
+ 'settings' => array(),
+ 'type' => 'i18n_taxonomy_term_reference_link',
+ 'weight' => 45,
+ ),
+ ),
+ 'entity_type' => 'scald_atom',
+ 'field_name' => 'field_video_opa_genre',
+ 'label' => 'OPA Genre',
+ 'required' => 0,
+ 'settings' => array(
+ 'entity_translation_sync' => FALSE,
+ 'user_register_form' => FALSE,
+ ),
+ 'widget' => array(
+ 'active' => 1,
+ 'module' => 'options',
+ 'settings' => array(),
+ 'type' => 'options_buttons',
+ 'weight' => 31,
+ ),
+ );
+
// Exported field_instance:
// 'taxonomy_term-foundation_opa_genres-field_opa_code'.
$field_instances['taxonomy_term-foundation_opa_genres-field_opa_code'] = array(
@@ -53,6 +85,7 @@ function taxonomy_opa_migrate_field_default_field_instances() {
// Translatables
// Included for use with string extractors like potx.
t('Code');
+ t('OPA Genre');
return $field_instances;
}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
index d9c9ae9..ca6b8d4 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
@@ -5,12 +5,16 @@ package = ARTE Foundation
dependencies[] = arte_opa
dependencies[] = ctools
dependencies[] = features
+dependencies[] = i18n_taxonomy
+dependencies[] = options
dependencies[] = strongarm
dependencies[] = taxonomy
dependencies[] = text
features[ctools][] = strongarm:strongarm:1
features[features_api][] = api:2
features[field_base][] = field_opa_code
+features[field_base][] = field_video_opa_genre
+features[field_instance][] = scald_atom-video-field_video_opa_genre
features[field_instance][] = taxonomy_term-foundation_opa_genres-field_opa_code
features[taxonomy][] = foundation_opa_genres
; OPA Classes
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
index a169818..00e5d5f 100644
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
+++ b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
@@ -5,3 +5,20 @@
*/
include_once 'taxonomy_opa_migrate.features.inc';
+
+/**
+ * Implements hook_scald_opa_migrate_mapping_info().
+ *
+ * Provide specific mappings of OPA categories terms into Drupal taxonomies.
+ *
+ */
+function taxonomy_opa_migrate_scald_opa_migrate_mapping_info() {
+ return array(
+ 'opa_taxonomy' => array(
+ array(
+ 'field_name' => 'field_video_opa_genre',
+ 'vocabulary_name' => 'foundation_opa_genres',
+ ),
+ ),
+ );
+}
From 7d8c4ee15325f85e4d424dda647c792bfcc8dc98 Mon Sep 17 00:00:00 2001
From: lsantos-oq <luis.santos@openquest.pt>
Date: Fri, 27 May 2016 17:35:09 +0100
Subject: [PATCH 06/12] AFOUND-731 Change the scald_opa_migrate to sync the OPA
Categories
---
.../custom/scald_opa_migrate/migrate/scald_opa.inc | 69 ++++++++++++++++++++++
1 file changed, 69 insertions(+)
diff --git a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
index 96936ff..7b3273e 100644
--- a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
+++ b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
@@ -174,6 +174,9 @@ class ScaldOPAMigration extends Migration {
// Parse thesaurus logic. This maps thesaurus terms on Drupal taxonomy terms
$this->parseThesaurus($row);
+
+ // Parse categories logic. This maps categories terms on Drupal taxonomy terms
+ $this->parseCategories($row);
}
/**
@@ -436,6 +439,72 @@ class ScaldOPAMigration extends Migration {
}
/**
+ * Parse categories terms. We use the hook_scald_opa_migrate_mapping_info
+ * to allow other modules to provide opa_taxonomy mapping settings.
+ *
+ * @param [type] $row [description]
+ * @return [type] [description]
+ */
+ protected function parseCategories($row) {
+ if (isset($this->customMappings['opa_taxonomy'])) {
+ $mappings = $this->customMappings['opa_taxonomy'];
+
+ // Each mapping provides a taxonomy - thesaurus mapping.
+ foreach ($mappings as $mapping) {
+ if (!empty($row->categories)) {
+ $categories_import = array();
+
+ // Loop through all thesaurus terms and save the terms we should import
+ foreach ($row->categories as $category) {
+ $categories_import[] = $category;
+ }
+
+ // Sync up terms, and take the returned tids
+ // We also make sure to include the parents
+ $key = $mapping['field_name'] . '_tids';
+ $row->$key = $this->fetchCategories($mapping['vocabulary_name'], $categories_import);
+
+ // Add the taxonomy mappings
+ if (!empty($row->$key)) {
+ $this->addFieldMapping($mapping['field_name'], $key, FALSE);
+ $this->addFieldMapping($mapping['field_name'] . ':source_type', NULL, FALSE)
+ ->defaultValue(TRUE);
+ }
+ else {
+ $this->removeFieldMapping($mapping['field_name']);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Get tid from OPA Categories.
+ *
+ * @param [type] $vocabulary_name [description]
+ * @param [type] $categories [description]
+ */
+ protected function fetchCategories($vocabulary_name, $categories) {
+ $tids = array();
+
+ foreach ($categories as $category) {
+ $query = new EntityFieldQuery();
+ $query = $query->entityCondition('entity_type', 'taxonomy_term')
+ ->entityCondition('bundle', $vocabulary_name)
+ ->fieldCondition('field_opa_code', 'value', $category['code'], '=');
+ $result = $query->execute();
+
+ if (isset($result['taxonomy_term'])) {
+ $terms = $result['taxonomy_term'];
+ $keys = array_keys($terms);
+ $tids = array_merge($tids, $keys);
+ }
+ }
+
+ return $tids;
+ }
+
+ /**
* Helper function to sort terms comping from OPA. Used in PHP usort.
*
* @param [type] $a [description]
From 88588be74ee7e2fe77211a7318adfe40934dc88d Mon Sep 17 00:00:00 2001
From: lsantos-oq <luis.santos@openquest.pt>
Date: Wed, 15 Jun 2016 17:04:16 +0100
Subject: [PATCH 07/12] AFOUND-731 Rename taxonomy to categories
---
modules/custom/categories_opa_migrate/README.md | 30 +++++
.../categories_opa_migrate.drush.inc | 128 ++++++++++++++++++++
.../categories_opa_migrate.features.field_base.inc | 70 +++++++++++
...egories_opa_migrate.features.field_instance.inc | 91 +++++++++++++++
.../categories_opa_migrate.features.inc | 14 +++
.../categories_opa_migrate.features.taxonomy.inc | 38 ++++++
.../categories_opa_migrate.info | 25 ++++
.../categories_opa_migrate.module | 24 ++++
.../includes/opa_categories_reader.inc | 79 +++++++++++++
.../migrate/categories_opa_migrate.inc | 129 +++++++++++++++++++++
.../migrate/sources/categories_opa.inc | 102 ++++++++++++++++
.../custom/scald_opa_migrate/migrate/scald_opa.inc | 6 +-
modules/custom/taxonomy_opa_migrate/README.md | 30 -----
.../includes/opa_categories_reader.inc | 79 -------------
.../migrate/sources/taxonomy_opa.inc | 102 ----------------
.../migrate/taxonomy_opa_migrate.inc | 129 ---------------------
.../taxonomy_opa_migrate.drush.inc | 128 --------------------
.../taxonomy_opa_migrate.features.field_base.inc | 70 -----------
...axonomy_opa_migrate.features.field_instance.inc | 91 ---------------
.../taxonomy_opa_migrate.features.inc | 14 ---
.../taxonomy_opa_migrate.features.taxonomy.inc | 38 ------
.../taxonomy_opa_migrate/taxonomy_opa_migrate.info | 25 ----
.../taxonomy_opa_migrate.module | 24 ----
23 files changed, 733 insertions(+), 733 deletions(-)
create mode 100644 modules/custom/categories_opa_migrate/README.md
create mode 100644 modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc
create mode 100644 modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc
create mode 100644 modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
create mode 100644 modules/custom/categories_opa_migrate/categories_opa_migrate.features.inc
create mode 100644 modules/custom/categories_opa_migrate/categories_opa_migrate.features.taxonomy.inc
create mode 100644 modules/custom/categories_opa_migrate/categories_opa_migrate.info
create mode 100644 modules/custom/categories_opa_migrate/categories_opa_migrate.module
create mode 100644 modules/custom/categories_opa_migrate/includes/opa_categories_reader.inc
create mode 100644 modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
create mode 100644 modules/custom/categories_opa_migrate/migrate/sources/categories_opa.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/README.md
delete mode 100644 modules/custom/taxonomy_opa_migrate/includes/opa_categories_reader.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.taxonomy.inc
delete mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
delete mode 100644 modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
diff --git a/modules/custom/categories_opa_migrate/README.md b/modules/custom/categories_opa_migrate/README.md
new file mode 100644
index 0000000..c35dd6c
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/README.md
@@ -0,0 +1,30 @@
+Categories OPA Migrate
+=================
+
+Module providing support for migrating from OPA Categories into `taxonomy terms`. Dynamic migrations use the migrate framework and Drush commands are supported to run migrations via the command-line.
+
+The Taxonomy that you select to put the migrate items need have the Translation mode: `Localize. Terms are common for all languages, but their name and description may be localized.`
+
+### Configuring OPA
+
+Follow configuration instructions in the `arte_opa` module to make sure you can connect to OPA.
+
+### Mapping
+
+See the Migrate class `categories_opa.inc` for mapping information.
+
+### Sources
+
+#### Full OPA queries
+
+The default configuration is querying OPA directly with the given parameters (see "Drush commands" for the parameters to pass on).
+
+### Drush commands
+
+The module provides a custom Drush command for synchronization
+
+ drush cos
+
+See `drush help tos` for examples. For example see examples
+
+ drush tos --platform=ALW
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc
new file mode 100644
index 0000000..00f80cf
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc
@@ -0,0 +1,128 @@
+<?php
+/**
+ * @file
+ *
+ * Provides drush extensions for Categories OPA
+ */
+
+/**
+ * Implements hook_drush_command().
+ */
+function categories_opa_migrate_drush_command() {
+ $items = array();
+
+ $items['categories-opa-sync'] = array(
+ 'description' => 'Import categories from OPA into Taxonomy terms',
+ 'aliases' => array('cos'),
+ 'options' => array(
+ 'platform' => 'The platform, e.g. ALW or AC. This is an obligatory option',
+ 'query' => 'E.g. "sort=language". Should have the format of URL GET parameters, and it might overwrite previously set parameters such as platform, code or language.',
+ 'limit' => 'Limit on the length of each migration process, expressed in seconds or number of items',
+ 'feedback' => 'Provides feedback during migration, see migration feedback option for more information',
+ 'debug-opa' => 'Output debugging information about the OPA requests issued during synchronization',
+ ),
+ 'examples' => array(
+ 'tos --platform=ALW --feedback="10 seconds"' => 'Imports the categories to the Foundation genres Taxonomy from the ALW platform having feedback every 10 seconds',
+ ),
+ );
+
+ return $items;
+}
+
+/**
+ * Command callback: run Categories OPA synchronization
+ */
+function drush_categories_opa_migrate_categories_opa_sync($args = NULL) {
+ // Fetch the platform - this option is obligatory and will fail if not provided
+ $platform = drush_get_option('platform', FALSE);
+ if (!$platform || $platform == 1) {
+ drush_log(dt('Missing or empty parameter --platform'), 'error');
+ return;
+ }
+
+ $arguments = array(
+ //'needsUpdate' => drush_get_option('update', FALSE),
+ 'debug' => drush_get_option('debug-opa', FALSE),
+ //'uniqueEmNumber' => drush_get_option('uniqueEmNumber', FALSE),
+ );
+
+
+ // Default query arguments
+ $arguments['query'] = array(
+ 'platform' => drush_get_option('platform'),
+ 'kind' => 'INTERNET',
+ 'language' => variable_get('categories_opa_migrate_base_language', 'en'),
+ );
+
+ // Set the taxonomy bundle
+ $arguments['bundle'] = 'foundation_opa_genres';
+
+ // Additional query parameters
+ if ($query_additional = drush_get_option('query', FALSE)) {
+ $parts = explode('&', $query_additional);
+ if (!empty($parts)) {
+ foreach ($parts as $part) {
+ $elements = explode('=', $part);
+ if (count($elements) == 2) {
+ $arguments['query'][$elements[0]] = $elements[1];
+ }
+ }
+ }
+ }
+
+ // Limit
+ if ($limit = drush_get_option('limit')) {
+ $options['limit']['value'] = $limit;
+ $options['limit']['unit'] = 'items';
+ }
+
+ // Feedback
+ $feedback = drush_get_option('feedback');
+ if ($feedback) {
+ $parts = explode(' ', $feedback);
+ $options['feedback']['value'] = $parts[0];
+ $options['feedback']['unit'] = $parts[1];
+ if ($options['feedback']['unit'] != 'seconds' &&
+ $options['feedback']['unit'] != 'second' &&
+ $options['feedback']['unit'] != 'items' &&
+ $options['feedback']['unit'] != 'item') {
+ drush_set_error(NULL, dt("Invalid feedback frequency unit '!unit'",
+ array('!unit' => $options['feedback']['unit'])));
+ return;
+ }
+ }
+
+ // Run the migration
+ _drush_categories_opa_migrate_run_migrate($arguments, $options);
+}
+
+/**
+ *
+ */
+function _drush_categories_opa_migrate_run_migrate($arguments = array(), $options = array()) {
+ $machine_name = 'CategoriesOPA';
+ Migration::registerMigration(
+ 'CategoriesOPAMigration',
+ $machine_name,
+ $arguments + array('group_name' => 'opa')
+ );
+
+ $migration = Migration::getInstance($machine_name);
+
+ // We will update nodes as we go along.
+ $migration->prepareUpdate();
+
+ // Import!
+ $status = $migration->processImport($options);
+
+ // Reset all caches, if not recreating our migration class with different
+ // parameters won't make any difference
+ drupal_static_reset();
+
+ // Clear cache and force refresh of JavaScript translations.
+ _locale_invalidate_js($langcode);
+ cache_clear_all('locale:', 'cache', TRUE);
+
+ // Rebuild the menu, strings may have changed.
+ menu_rebuild();
+}
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc
new file mode 100644
index 0000000..378efa4
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc
@@ -0,0 +1,70 @@
+<?php
+/**
+ * @file
+ * categories_opa_migrate.features.field_base.inc
+ */
+
+/**
+ * Implements hook_field_default_field_bases().
+ */
+function categories_opa_migrate_field_default_field_bases() {
+ $field_bases = array();
+
+ // Exported field_base: 'field_opa_code'.
+ $field_bases['field_opa_code'] = array(
+ 'active' => 1,
+ 'cardinality' => 1,
+ 'deleted' => 0,
+ 'entity_types' => array(),
+ 'field_name' => 'field_opa_code',
+ 'field_permissions' => array(
+ 'type' => 0,
+ ),
+ 'indexes' => array(
+ 'format' => array(
+ 0 => 'format',
+ ),
+ ),
+ 'locked' => 0,
+ 'module' => 'text',
+ 'settings' => array(
+ 'entity_translation_sync' => FALSE,
+ 'max_length' => 60,
+ ),
+ 'translatable' => 0,
+ 'type' => 'text',
+ );
+
+ // Exported field_base: 'field_video_opa_genre'.
+ $field_bases['field_video_opa_genre'] = array(
+ 'active' => 1,
+ 'cardinality' => -1,
+ 'deleted' => 0,
+ 'entity_types' => array(),
+ 'field_name' => 'field_video_opa_genre',
+ 'field_permissions' => array(
+ 'type' => 0,
+ ),
+ 'indexes' => array(
+ 'tid' => array(
+ 0 => 'tid',
+ ),
+ ),
+ 'locked' => 0,
+ 'module' => 'taxonomy',
+ 'settings' => array(
+ 'allowed_values' => array(
+ 0 => array(
+ 'vocabulary' => 'foundation_opa_genres',
+ 'parent' => 0,
+ ),
+ ),
+ 'entity_translation_sync' => FALSE,
+ 'options_list_callback' => 'i18n_taxonomy_allowed_values',
+ ),
+ 'translatable' => 0,
+ 'type' => 'taxonomy_term_reference',
+ );
+
+ return $field_bases;
+}
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
new file mode 100644
index 0000000..69a586d
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
@@ -0,0 +1,91 @@
+<?php
+/**
+ * @file
+ * categories_opa_migrate.features.field_instance.inc
+ */
+
+/**
+ * Implements hook_field_default_field_instances().
+ */
+function categories_opa_migrate_field_default_field_instances() {
+ $field_instances = array();
+
+ // Exported field_instance: 'scald_atom-video-field_video_opa_genre'.
+ $field_instances['scald_atom-video-field_video_opa_genre'] = array(
+ 'bundle' => 'video',
+ 'default_value' => NULL,
+ 'deleted' => 0,
+ 'description' => '',
+ 'display' => array(
+ 'default' => array(
+ 'label' => 'above',
+ 'module' => 'i18n_taxonomy',
+ 'settings' => array(),
+ 'type' => 'i18n_taxonomy_term_reference_link',
+ 'weight' => 45,
+ ),
+ ),
+ 'entity_type' => 'scald_atom',
+ 'field_name' => 'field_video_opa_genre',
+ 'label' => 'OPA Genre',
+ 'required' => 0,
+ 'settings' => array(
+ 'entity_translation_sync' => FALSE,
+ 'user_register_form' => FALSE,
+ ),
+ 'widget' => array(
+ 'active' => 1,
+ 'module' => 'options',
+ 'settings' => array(),
+ 'type' => 'options_buttons',
+ 'weight' => 31,
+ ),
+ );
+
+ // Exported field_instance:
+ // 'taxonomy_term-foundation_opa_genres-field_opa_code'.
+ $field_instances['taxonomy_term-foundation_opa_genres-field_opa_code'] = array(
+ 'bundle' => 'foundation_opa_genres',
+ 'default_value' => NULL,
+ 'deleted' => 0,
+ 'description' => '',
+ 'display' => array(
+ 'default' => array(
+ 'label' => 'above',
+ 'module' => 'text',
+ 'settings' => array(),
+ 'type' => 'text_default',
+ 'weight' => 0,
+ ),
+ ),
+ 'entity_type' => 'taxonomy_term',
+ 'field_name' => 'field_opa_code',
+ 'label' => 'Code',
+ 'required' => 1,
+ 'settings' => array(
+ 'context' => 'title',
+ 'context_default' => 'sdl_editor_representation',
+ 'dnd_enabled' => 0,
+ 'entity_translation_sync' => FALSE,
+ 'mee_enabled' => 0,
+ 'text_processing' => 0,
+ 'user_register_form' => FALSE,
+ ),
+ 'widget' => array(
+ 'active' => 1,
+ 'module' => 'text',
+ 'settings' => array(
+ 'size' => 60,
+ ),
+ 'type' => 'text_textfield',
+ 'weight' => 41,
+ ),
+ );
+
+ // Translatables
+ // Included for use with string extractors like potx.
+ t('Code');
+ t('OPA Genre');
+
+ return $field_instances;
+}
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.inc
new file mode 100644
index 0000000..10b676a
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.inc
@@ -0,0 +1,14 @@
+<?php
+/**
+ * @file
+ * categories_opa_migrate.features.inc
+ */
+
+/**
+ * Implements hook_ctools_plugin_api().
+ */
+function categories_opa_migrate_ctools_plugin_api($module = NULL, $api = NULL) {
+ if ($module == "strongarm" && $api == "strongarm") {
+ return array("version" => "1");
+ }
+}
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.taxonomy.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.taxonomy.inc
new file mode 100644
index 0000000..bdfb6a0
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.taxonomy.inc
@@ -0,0 +1,38 @@
+<?php
+/**
+ * @file
+ * categories_opa_migrate.features.taxonomy.inc
+ */
+
+/**
+ * Implements hook_taxonomy_default_vocabularies().
+ */
+function categories_opa_migrate_taxonomy_default_vocabularies() {
+ return array(
+ 'foundation_opa_genres' => array(
+ 'name' => 'Foundation Genres from OPA',
+ 'machine_name' => 'foundation_opa_genres',
+ 'description' => '',
+ 'hierarchy' => 0,
+ 'module' => 'taxonomy',
+ 'weight' => 0,
+ 'language' => 'und',
+ 'i18n_mode' => 1,
+ 'rdf_mapping' => array(
+ 'rdftype' => array(
+ 0 => 'skos:ConceptScheme',
+ ),
+ 'name' => array(
+ 'predicates' => array(
+ 0 => 'dc:title',
+ ),
+ ),
+ 'description' => array(
+ 'predicates' => array(
+ 0 => 'rdfs:comment',
+ ),
+ ),
+ ),
+ ),
+ );
+}
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.info b/modules/custom/categories_opa_migrate/categories_opa_migrate.info
new file mode 100644
index 0000000..1d7fe8b
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.info
@@ -0,0 +1,25 @@
+name = Categories OPA Migrate
+description = Provides Categories from OPA.
+core = 7.x
+package = ARTE Foundation
+dependencies[] = arte_opa
+dependencies[] = ctools
+dependencies[] = features
+dependencies[] = i18n_taxonomy
+dependencies[] = options
+dependencies[] = strongarm
+dependencies[] = taxonomy
+dependencies[] = text
+features[ctools][] = strongarm:strongarm:1
+features[features_api][] = api:2
+features[field_base][] = field_opa_code
+features[field_base][] = field_video_opa_genre
+features[field_instance][] = scald_atom-video-field_video_opa_genre
+features[field_instance][] = taxonomy_term-foundation_opa_genres-field_opa_code
+features[taxonomy][] = foundation_opa_genres
+; OPA Classes
+files[] = includes/opa_categories_reader.inc
+; Migrate source classes
+files[] = migrate/sources/categories_opa.inc
+; Migrate classes
+files[] = migrate/categories_opa_migrate.inc
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.module b/modules/custom/categories_opa_migrate/categories_opa_migrate.module
new file mode 100644
index 0000000..814e299
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.module
@@ -0,0 +1,24 @@
+<?php
+/**
+ * @file
+ * Code for the categories_opa_migrate feature.
+ */
+
+include_once 'categories_opa_migrate.features.inc';
+
+/**
+ * Implements hook_scald_opa_migrate_mapping_info().
+ *
+ * Provide specific mappings of OPA categories terms into Drupal taxonomies.
+ *
+ */
+function categories_opa_migrate_scald_opa_migrate_mapping_info() {
+ return array(
+ 'opa_categories' => array(
+ array(
+ 'field_name' => 'field_video_opa_genre',
+ 'vocabulary_name' => 'foundation_opa_genres',
+ ),
+ ),
+ );
+}
diff --git a/modules/custom/categories_opa_migrate/includes/opa_categories_reader.inc b/modules/custom/categories_opa_migrate/includes/opa_categories_reader.inc
new file mode 100644
index 0000000..67f079a
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/includes/opa_categories_reader.inc
@@ -0,0 +1,79 @@
+<?php
+
+/**
+ * @file
+ * OPA Categories reader
+ */
+
+class OPACategoriesReader extends OPAReader {
+
+ /**
+ * Initialize the members.
+ */
+ public function __construct($opa, $params) {
+ parent::__construct($opa, $params, 'code', 'categories');
+ }
+
+ /**
+ * Implementation of Iterator::next().
+ *
+ * Populates currentElement (the object being retrieved) and currentId (that
+ * object's unique identifier) from the specified JSON file. Sets both to
+ * NULL at end-of-file. Handles properly-formed JSON, as well as some improper
+ * coding (specifically that generated in Ning exports).
+ *
+ * @return void
+ */
+ public function next() {
+ // Use pagination when no user limit was specified or when the specified limit is
+ // bigger than the maximum allowed by OPA
+ $pagination = is_null($this->limit) || $this->limit > $this->pagination_limit;
+
+ // Open the file and position it if necessary
+ if (empty($this->data)) {
+ $options = $this->params;
+ $options['page'] = $this->page;
+ $options['limit'] = $pagination ? $this->pagination_limit : $this->limit;
+
+ // API method callback with pagination and limits
+ $this->data = $this->opa->apiCall($this->resourcePath(), $options);
+
+ // @todo add try/catch
+ if (!$this->data) {
+ return FALSE;
+ }
+ }
+
+ // Increment Id
+ $this->position++;
+
+ // If we're paginating with a user-specified limit, check if the iterator has reached the limit
+ //
+ // NB the last OPA page will contain the pagination limit, i.e. 50 items by default.
+ // If however the user-specified limit is 60 for instance, we only want the remaining 10 items from
+ // that last page.
+ $last_page_delta = $this->limit - (($this->page - 1) * $this->pagination_limit);
+ if (!is_null($this->limit) && $pagination && $this->position + 1 > $last_page_delta) {
+ // We are done!
+ $this->currentElement = NULL;
+ }
+ elseif (isset($this->data[$this->resource][$this->position])) {
+ $this->currentElement = $this->getData();
+ $this->currentId = $this->currentElement['code'];
+ }
+ else {
+ // Check if we need to advance the pagination
+ if (isset($this->data['meta'][$this->resource]['pages']) && $pagination && $this->page < $this->data['meta'][$this->resource]['pages']) {
+ // Yes, and we have pages available still, so increment page and reset data to fetch the next page
+ $this->page++;
+ $this->position = -1;
+ $this->data = array();
+ $this->next();
+ }
+ else {
+ // We are done!
+ $this->currentElement = NULL;
+ }
+ }
+ }
+}
diff --git a/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc b/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
new file mode 100644
index 0000000..5e1e730
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
@@ -0,0 +1,129 @@
+<?php
+
+/**
+ * @file
+ * Migration class for Categories OPA migrations.
+ */
+
+class CategoriesOPAMigration extends Migration {
+ protected $key = 'code';
+
+ // Custom mappings as provided by modules implementing hook_categories_opa_migrate_mapping_info();
+ protected $customMappings = array();
+
+ // OPA connection
+ protected $opa = null;
+
+ // The translation query
+ protected $translationsQuery = array();
+
+ // Internal store of newly created or updated terms objects
+ protected $processedTerms = array();
+
+ public function __construct($arguments) {
+ parent::__construct($arguments);
+
+ $this->description = t('Import categories into Taxonomy terms.');
+
+ // Create a map object for tracking the relationships between source rows.
+ // If we want to enforce unique emission numbers also outside of this migration,
+ // we will use the MigrateUniqueEMNumberSQLMap class instead of the default
+ // MigrateSQLMap class
+ $mapClass = 'MigrateSQLMap';
+ if (isset($arguments['uniqueEmNumber']) && $arguments['uniqueEmNumber']) {
+ $mapClass = 'MigrateUniqueEMNumberSQLMap';
+ }
+
+ $this->map = new $mapClass(
+ $this->machineName,
+ array(
+ $this->key => array(
+ 'type' => 'varchar',
+ 'length' => 255,
+ ),
+ ),
+ MigrateDestinationTerm::getKeySchema()
+ );
+
+ // Destination
+ $this->destination = new MigrateDestinationTerm($arguments['bundle']);
+
+ // Get an OPA connection
+ $this->opa = arte_opa_get_api();
+
+ // Use debug configuration for OPA?
+ if (isset($arguments['debug']) && $arguments['debug']) {
+ $opa->setDebug();
+ }
+
+ // Source: OPA with parameters
+ $query = isset($arguments['query']) ? $arguments['query'] : array();
+ $this->source = new MigrateSourceCategoriesOPA($this->opa, $query);
+
+ // Build the translation query
+ $this->translationsQuery = isset($arguments['query']) ? $arguments['query'] : array();
+ $languages_enabled = arte_opa_languages();
+
+ if (in_array($arguments['query']['language'], $languages_enabled)) {
+ unset($languages_enabled[$arguments['query']['language']]);
+ }
+ $this->translationsQuery['language'] = implode(',', $languages_enabled);
+
+ // Field mappings
+ $this->addFieldMapping('name', 'label');
+ $this->addFieldMapping('weight', 'order');
+ $this->addFieldMapping('field_opa_code', 'code');
+
+ // Grab custom mappings provided by other modules.
+ $this->customMappings = module_invoke_all('categories_opa_migrate_mapping_info');
+ }
+
+ public function prepareRow($row) {
+ if (parent::prepareRow($row) === FALSE) {
+ return FALSE;
+ }
+ }
+
+ /**
+ * Called immediately after the complete destination object is saved.
+ */
+ public function complete($entity, $row) {
+ // Find out if this was a new taxonomy term or the updated of an existing one
+ $hook = isset($entity->original) ? 'update' : 'insert';
+
+ // Logging
+ $msg = 'Taxonomy term %verb: %no_emiarte %kind, %title.';
+ $vars = array(
+ '%verb' => isset($entity->original) ? 'updated' : 'created',
+ '%no_emiarte' => $entity->base_id,
+ '%kind' => $row->kind,
+ '%title' => $entity->title,
+ );
+ $link = l('Taxonomy term ' . $entity->tid, 'taxonomy/term/' . $entity->tid);
+ watchdog('categories_opa_migrate', $msg, $vars, WATCHDOG_DEBUG, $link);
+
+ // Store processed atoms
+ $this->processedTerms[] = $entity;
+
+ // Get the translations
+ $query = $this->translationsQuery;
+ $query['code'] = $row->code;
+ $i18n_category = new MigrateSourceCategoriesOPA($this->opa, $query);
+
+ while ($i18n_row = $i18n_category->getNextRow()) {
+ $report = array();
+ _locale_import_one_string_db($report, $i18n_row->language, 'term:'. $entity->tid .':name', $row->label, $i18n_row->label, 'Taxonomy', '', LOCALE_IMPORT_OVERWRITE);
+ }
+
+ // Notify that we have a new or updated scald atom.
+ module_invoke_all('categories_opa_migrate_' . $hook, $entity);
+ }
+
+ /**
+ * Called when the import has finished
+ */
+ public function postImport() {
+ // Notify that the import has finished
+ module_invoke_all('categories_opa_migrate_complete', $this->processedTerms);
+ }
+}
diff --git a/modules/custom/categories_opa_migrate/migrate/sources/categories_opa.inc b/modules/custom/categories_opa_migrate/migrate/sources/categories_opa.inc
new file mode 100644
index 0000000..3529bba
--- /dev/null
+++ b/modules/custom/categories_opa_migrate/migrate/sources/categories_opa.inc
@@ -0,0 +1,102 @@
+<?php
+
+/**
+ * @file
+ * Support for the OPA source, accepting an OPA class and parameters to make
+ * connections.
+ */
+class MigrateSourceCategoriesOPA extends MigrateSource {
+
+ protected $reader;
+
+ /**
+ * Source constructor.
+ */
+ public function __construct($opa, $params, array $fields = array(), array $options = array()) {
+ parent::__construct($options);
+
+ $this->fields = $fields;
+
+ // Setup our reader
+ $this->reader = new OPACategoriesReader($opa, $params);
+ }
+
+ /**
+ * Return a string representing OPA
+ */
+ public function __toString() {
+ return (string) $this->reader;
+ }
+
+ /**
+ * Returns a list of fields available to be mapped from the source query.
+ *
+ * @return array
+ * Keys: machine names of the fields (to be passed to addFieldMapping)
+ * Values: Human-friendly descriptions of the fields.
+ */
+ public function fields() {
+ return $this->fields;
+ }
+
+ /**
+ * Return a count of all available source records.
+ */
+ public function computeCount() {
+ try {
+ return count($this->reader);
+ }
+ catch (OPAException $e) {
+ return 0;
+ }
+ }
+
+ /**
+ * Implementation of MigrateSource::performRewind().
+ */
+ public function performRewind() {
+ $this->reader->rewind();
+ }
+
+ /**
+ * Implementation of MigrationSource::getNextRow().
+ *
+ * @return stdClass
+ * data for the next row from the JSON source files
+ */
+ public function getNextRow() {
+ migrate_instrument_start('MigrateSourceOPA::next');
+
+ $next = NULL;
+
+ if (!isset($next) && $this->reader->valid()) {
+ $next = (object)$this->reader->current();
+ }
+
+ if (!isset($next)) {
+ return;
+ }
+
+ $row = $next;
+ do {
+ $this->reader->next();
+
+ if ($this->reader->valid()) {
+ $next = (object)$this->reader->current();
+
+ // Keep looping on our results if $row and $next have the
+ // same primary key
+ $same = ($row->code == $next->code && $row->kind == $next->kind && $row->language == $next->language);
+ }
+ else {
+ $next = NULL;
+ $same = FALSE;
+ }
+ }
+ while ($same && $next);
+
+ migrate_instrument_stop('MigrateSourceOPA::next');
+
+ return $row;
+ }
+}
diff --git a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
index 7b3273e..3338e27 100644
--- a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
+++ b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
@@ -440,14 +440,14 @@ class ScaldOPAMigration extends Migration {
/**
* Parse categories terms. We use the hook_scald_opa_migrate_mapping_info
- * to allow other modules to provide opa_taxonomy mapping settings.
+ * to allow other modules to provide opa_categories mapping settings.
*
* @param [type] $row [description]
* @return [type] [description]
*/
protected function parseCategories($row) {
- if (isset($this->customMappings['opa_taxonomy'])) {
- $mappings = $this->customMappings['opa_taxonomy'];
+ if (isset($this->customMappings['opa_categories'])) {
+ $mappings = $this->customMappings['opa_categories'];
// Each mapping provides a taxonomy - thesaurus mapping.
foreach ($mappings as $mapping) {
diff --git a/modules/custom/taxonomy_opa_migrate/README.md b/modules/custom/taxonomy_opa_migrate/README.md
deleted file mode 100644
index 58d67b0..0000000
--- a/modules/custom/taxonomy_opa_migrate/README.md
+++ /dev/null
@@ -1,30 +0,0 @@
-Taxonomy OPA Migrate
-=================
-
-Module providing support for migrating from OPA into `taxonomy terms`. Dynamic migrations use the migrate framework and Drush commands are supported to run migrations via the command-line.
-
-The Taxonomy that you select to put the migrate items need have the Translation mode: `Localize. Terms are common for all languages, but their name and description may be localized.`
-
-### Configuring OPA
-
-Follow configuration instructions in the `arte_opa` module to make sure you can connect to OPA.
-
-### Mapping
-
-See the Migrate class `taxonomy_opa.inc` for mapping information.
-
-### Sources
-
-#### Full OPA queries
-
-The default configuration is querying OPA directly with the given parameters (see "Drush commands" for the parameters to pass on).
-
-### Drush commands
-
-The module provides a custom Drush command for synchronization
-
- drush tos
-
-See `drush help tos` for examples. For example see examples
-
- drush tos --platform=ALW
diff --git a/modules/custom/taxonomy_opa_migrate/includes/opa_categories_reader.inc b/modules/custom/taxonomy_opa_migrate/includes/opa_categories_reader.inc
deleted file mode 100644
index 67f079a..0000000
--- a/modules/custom/taxonomy_opa_migrate/includes/opa_categories_reader.inc
+++ /dev/null
@@ -1,79 +0,0 @@
-<?php
-
-/**
- * @file
- * OPA Categories reader
- */
-
-class OPACategoriesReader extends OPAReader {
-
- /**
- * Initialize the members.
- */
- public function __construct($opa, $params) {
- parent::__construct($opa, $params, 'code', 'categories');
- }
-
- /**
- * Implementation of Iterator::next().
- *
- * Populates currentElement (the object being retrieved) and currentId (that
- * object's unique identifier) from the specified JSON file. Sets both to
- * NULL at end-of-file. Handles properly-formed JSON, as well as some improper
- * coding (specifically that generated in Ning exports).
- *
- * @return void
- */
- public function next() {
- // Use pagination when no user limit was specified or when the specified limit is
- // bigger than the maximum allowed by OPA
- $pagination = is_null($this->limit) || $this->limit > $this->pagination_limit;
-
- // Open the file and position it if necessary
- if (empty($this->data)) {
- $options = $this->params;
- $options['page'] = $this->page;
- $options['limit'] = $pagination ? $this->pagination_limit : $this->limit;
-
- // API method callback with pagination and limits
- $this->data = $this->opa->apiCall($this->resourcePath(), $options);
-
- // @todo add try/catch
- if (!$this->data) {
- return FALSE;
- }
- }
-
- // Increment Id
- $this->position++;
-
- // If we're paginating with a user-specified limit, check if the iterator has reached the limit
- //
- // NB the last OPA page will contain the pagination limit, i.e. 50 items by default.
- // If however the user-specified limit is 60 for instance, we only want the remaining 10 items from
- // that last page.
- $last_page_delta = $this->limit - (($this->page - 1) * $this->pagination_limit);
- if (!is_null($this->limit) && $pagination && $this->position + 1 > $last_page_delta) {
- // We are done!
- $this->currentElement = NULL;
- }
- elseif (isset($this->data[$this->resource][$this->position])) {
- $this->currentElement = $this->getData();
- $this->currentId = $this->currentElement['code'];
- }
- else {
- // Check if we need to advance the pagination
- if (isset($this->data['meta'][$this->resource]['pages']) && $pagination && $this->page < $this->data['meta'][$this->resource]['pages']) {
- // Yes, and we have pages available still, so increment page and reset data to fetch the next page
- $this->page++;
- $this->position = -1;
- $this->data = array();
- $this->next();
- }
- else {
- // We are done!
- $this->currentElement = NULL;
- }
- }
- }
-}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc b/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc
deleted file mode 100644
index 34be07e..0000000
--- a/modules/custom/taxonomy_opa_migrate/migrate/sources/taxonomy_opa.inc
+++ /dev/null
@@ -1,102 +0,0 @@
-<?php
-
-/**
- * @file
- * Support for the OPA source, accepting an OPA class and parameters to make
- * connections.
- */
-class MigrateSourceTaxonomyOPA extends MigrateSource {
-
- protected $reader;
-
- /**
- * Source constructor.
- */
- public function __construct($opa, $params, array $fields = array(), array $options = array()) {
- parent::__construct($options);
-
- $this->fields = $fields;
-
- // Setup our reader
- $this->reader = new OPACategoriesReader($opa, $params);
- }
-
- /**
- * Return a string representing OPA
- */
- public function __toString() {
- return (string) $this->reader;
- }
-
- /**
- * Returns a list of fields available to be mapped from the source query.
- *
- * @return array
- * Keys: machine names of the fields (to be passed to addFieldMapping)
- * Values: Human-friendly descriptions of the fields.
- */
- public function fields() {
- return $this->fields;
- }
-
- /**
- * Return a count of all available source records.
- */
- public function computeCount() {
- try {
- return count($this->reader);
- }
- catch (OPAException $e) {
- return 0;
- }
- }
-
- /**
- * Implementation of MigrateSource::performRewind().
- */
- public function performRewind() {
- $this->reader->rewind();
- }
-
- /**
- * Implementation of MigrationSource::getNextRow().
- *
- * @return stdClass
- * data for the next row from the JSON source files
- */
- public function getNextRow() {
- migrate_instrument_start('MigrateSourceOPA::next');
-
- $next = NULL;
-
- if (!isset($next) && $this->reader->valid()) {
- $next = (object)$this->reader->current();
- }
-
- if (!isset($next)) {
- return;
- }
-
- $row = $next;
- do {
- $this->reader->next();
-
- if ($this->reader->valid()) {
- $next = (object)$this->reader->current();
-
- // Keep looping on our results if $row and $next have the
- // same primary key
- $same = ($row->code == $next->code && $row->kind == $next->kind && $row->language == $next->language);
- }
- else {
- $next = NULL;
- $same = FALSE;
- }
- }
- while ($same && $next);
-
- migrate_instrument_stop('MigrateSourceOPA::next');
-
- return $row;
- }
-}
diff --git a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc b/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
deleted file mode 100644
index 80667d7..0000000
--- a/modules/custom/taxonomy_opa_migrate/migrate/taxonomy_opa_migrate.inc
+++ /dev/null
@@ -1,129 +0,0 @@
-<?php
-
-/**
- * @file
- * Migration class for Taxonomy OPA migrations.
- */
-
-class TaxonomyOPAMigration extends Migration {
- protected $key = 'code';
-
- // Custom mappings as provided by modules implementing hook_taxonomy_opa_migrate_mapping_info();
- protected $customMappings = array();
-
- // OPA connection
- protected $opa = null;
-
- // The translation query
- protected $translationsQuery = array();
-
- // Internal store of newly created or updated terms objects
- protected $processedTerms = array();
-
- public function __construct($arguments) {
- parent::__construct($arguments);
-
- $this->description = t('Import categories into Taxonomy terms.');
-
- // Create a map object for tracking the relationships between source rows.
- // If we want to enforce unique emission numbers also outside of this migration,
- // we will use the MigrateUniqueEMNumberSQLMap class instead of the default
- // MigrateSQLMap class
- $mapClass = 'MigrateSQLMap';
- if (isset($arguments['uniqueEmNumber']) && $arguments['uniqueEmNumber']) {
- $mapClass = 'MigrateUniqueEMNumberSQLMap';
- }
-
- $this->map = new $mapClass(
- $this->machineName,
- array(
- $this->key => array(
- 'type' => 'varchar',
- 'length' => 255,
- ),
- ),
- MigrateDestinationTerm::getKeySchema()
- );
-
- // Destination
- $this->destination = new MigrateDestinationTerm($arguments['bundle']);
-
- // Get an OPA connection
- $this->opa = arte_opa_get_api();
-
- // Use debug configuration for OPA?
- if (isset($arguments['debug']) && $arguments['debug']) {
- $opa->setDebug();
- }
-
- // Source: OPA with parameters
- $query = isset($arguments['query']) ? $arguments['query'] : array();
- $this->source = new MigrateSourceTaxonomyOPA($this->opa, $query);
-
- // Build the translation query
- $this->translationsQuery = isset($arguments['query']) ? $arguments['query'] : array();
- $languages_enabled = arte_opa_languages();
-
- if (in_array($arguments['query']['language'], $languages_enabled)) {
- unset($languages_enabled[$arguments['query']['language']]);
- }
- $this->translationsQuery['language'] = implode(',', $languages_enabled);
-
- // Field mappings
- $this->addFieldMapping('name', 'label');
- $this->addFieldMapping('weight', 'order');
- $this->addFieldMapping('field_opa_code', 'code');
-
- // Grab custom mappings provided by other modules.
- $this->customMappings = module_invoke_all('taxonomy_opa_migrate_mapping_info');
- }
-
- public function prepareRow($row) {
- if (parent::prepareRow($row) === FALSE) {
- return FALSE;
- }
- }
-
- /**
- * Called immediately after the complete destination object is saved.
- */
- public function complete($entity, $row) {
- // Find out if this was a new taxonomy term or the updated of an existing one
- $hook = isset($entity->original) ? 'update' : 'insert';
-
- // Logging
- $msg = 'Taxonomy term %verb: %no_emiarte %kind, %title.';
- $vars = array(
- '%verb' => isset($entity->original) ? 'updated' : 'created',
- '%no_emiarte' => $entity->base_id,
- '%kind' => $row->kind,
- '%title' => $entity->title,
- );
- $link = l('Taxonomy term ' . $entity->tid, 'taxonomy/term/' . $entity->tid);
- watchdog('taxonomy_opa_migrate', $msg, $vars, WATCHDOG_DEBUG, $link);
-
- // Store processed atoms
- $this->processedTerms[] = $entity;
-
- // Get the translations
- $query = $this->translationsQuery;
- $query['code'] = $row->code;
- $i18n_category = new MigrateSourceTaxonomyOPA($this->opa, $query);
-
- while ($i18n_row = $i18n_category->getNextRow()) {
- $report = array();
- _locale_import_one_string_db($report, $i18n_row->language, 'term:'. $entity->tid .':name', $row->label, $i18n_row->label, 'Taxonomy', '', LOCALE_IMPORT_OVERWRITE);
- }
-
- // Notify that we have a new or updated scald atom.
- module_invoke_all('taxonomy_opa_migrate_' . $hook, $entity);
- }
-
- /**
- * Called when the import has finished
- */
- public function postImport() {
- // Notify that the import has finished
- module_invoke_all('taxonomy_opa_migrate_complete', $this->processedTerms);
- }
-}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
deleted file mode 100644
index 19b083d..0000000
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.drush.inc
+++ /dev/null
@@ -1,128 +0,0 @@
-<?php
-/**
- * @file
- *
- * Provides drush extensions for Taxonomy OPA
- */
-
-/**
- * Implements hook_drush_command().
- */
-function taxonomy_opa_migrate_drush_command() {
- $items = array();
-
- $items['taxonomy-opa-sync'] = array(
- 'description' => 'Import categories from OPA into Taxonomy terms',
- 'aliases' => array('tos'),
- 'options' => array(
- 'platform' => 'The platform, e.g. ALW or AC. This is an obligatory option',
- 'query' => 'E.g. "sort=language". Should have the format of URL GET parameters, and it might overwrite previously set parameters such as platform, code or language.',
- 'limit' => 'Limit on the length of each migration process, expressed in seconds or number of items',
- 'feedback' => 'Provides feedback during migration, see migration feedback option for more information',
- 'debug-opa' => 'Output debugging information about the OPA requests issued during synchronization',
- ),
- 'examples' => array(
- 'tos --platform=ALW --feedback="10 seconds"' => 'Imports the categories to the Foundation genres Taxonomy from the ALW platform having feedback every 10 seconds',
- ),
- );
-
- return $items;
-}
-
-/**
- * Command callback: run Taxonomy OPA synchronization
- */
-function drush_taxonomy_opa_migrate_taxonomy_opa_sync($args = NULL) {
- // Fetch the platform - this option is obligatory and will fail if not provided
- $platform = drush_get_option('platform', FALSE);
- if (!$platform || $platform == 1) {
- drush_log(dt('Missing or empty parameter --platform'), 'error');
- return;
- }
-
- $arguments = array(
- //'needsUpdate' => drush_get_option('update', FALSE),
- 'debug' => drush_get_option('debug-opa', FALSE),
- //'uniqueEmNumber' => drush_get_option('uniqueEmNumber', FALSE),
- );
-
-
- // Default query arguments
- $arguments['query'] = array(
- 'platform' => drush_get_option('platform'),
- 'kind' => 'INTERNET',
- 'language' => variable_get('taxonomy_opa_migrate_base_language', 'en'),
- );
-
- // Set the taxonomy bundle
- $arguments['bundle'] = 'foundation_opa_genres';
-
- // Additional query parameters
- if ($query_additional = drush_get_option('query', FALSE)) {
- $parts = explode('&', $query_additional);
- if (!empty($parts)) {
- foreach ($parts as $part) {
- $elements = explode('=', $part);
- if (count($elements) == 2) {
- $arguments['query'][$elements[0]] = $elements[1];
- }
- }
- }
- }
-
- // Limit
- if ($limit = drush_get_option('limit')) {
- $options['limit']['value'] = $limit;
- $options['limit']['unit'] = 'items';
- }
-
- // Feedback
- $feedback = drush_get_option('feedback');
- if ($feedback) {
- $parts = explode(' ', $feedback);
- $options['feedback']['value'] = $parts[0];
- $options['feedback']['unit'] = $parts[1];
- if ($options['feedback']['unit'] != 'seconds' &&
- $options['feedback']['unit'] != 'second' &&
- $options['feedback']['unit'] != 'items' &&
- $options['feedback']['unit'] != 'item') {
- drush_set_error(NULL, dt("Invalid feedback frequency unit '!unit'",
- array('!unit' => $options['feedback']['unit'])));
- return;
- }
- }
-
- // Run the migration
- _drush_taxonomy_opa_migrate_run_migrate($arguments, $options);
-}
-
-/**
- *
- */
-function _drush_taxonomy_opa_migrate_run_migrate($arguments = array(), $options = array()) {
- $machine_name = 'TaxonomyOPA';
- Migration::registerMigration(
- 'TaxonomyOPAMigration',
- $machine_name,
- $arguments + array('group_name' => 'opa')
- );
-
- $migration = Migration::getInstance($machine_name);
-
- // We will update nodes as we go along.
- $migration->prepareUpdate();
-
- // Import!
- $status = $migration->processImport($options);
-
- // Reset all caches, if not recreating our migration class with different
- // parameters won't make any difference
- drupal_static_reset();
-
- // Clear cache and force refresh of JavaScript translations.
- _locale_invalidate_js($langcode);
- cache_clear_all('locale:', 'cache', TRUE);
-
- // Rebuild the menu, strings may have changed.
- menu_rebuild();
-}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc
deleted file mode 100644
index 1b16150..0000000
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_base.inc
+++ /dev/null
@@ -1,70 +0,0 @@
-<?php
-/**
- * @file
- * taxonomy_opa_migrate.features.field_base.inc
- */
-
-/**
- * Implements hook_field_default_field_bases().
- */
-function taxonomy_opa_migrate_field_default_field_bases() {
- $field_bases = array();
-
- // Exported field_base: 'field_opa_code'.
- $field_bases['field_opa_code'] = array(
- 'active' => 1,
- 'cardinality' => 1,
- 'deleted' => 0,
- 'entity_types' => array(),
- 'field_name' => 'field_opa_code',
- 'field_permissions' => array(
- 'type' => 0,
- ),
- 'indexes' => array(
- 'format' => array(
- 0 => 'format',
- ),
- ),
- 'locked' => 0,
- 'module' => 'text',
- 'settings' => array(
- 'entity_translation_sync' => FALSE,
- 'max_length' => 60,
- ),
- 'translatable' => 0,
- 'type' => 'text',
- );
-
- // Exported field_base: 'field_video_opa_genre'.
- $field_bases['field_video_opa_genre'] = array(
- 'active' => 1,
- 'cardinality' => -1,
- 'deleted' => 0,
- 'entity_types' => array(),
- 'field_name' => 'field_video_opa_genre',
- 'field_permissions' => array(
- 'type' => 0,
- ),
- 'indexes' => array(
- 'tid' => array(
- 0 => 'tid',
- ),
- ),
- 'locked' => 0,
- 'module' => 'taxonomy',
- 'settings' => array(
- 'allowed_values' => array(
- 0 => array(
- 'vocabulary' => 'foundation_opa_genres',
- 'parent' => 0,
- ),
- ),
- 'entity_translation_sync' => FALSE,
- 'options_list_callback' => 'i18n_taxonomy_allowed_values',
- ),
- 'translatable' => 0,
- 'type' => 'taxonomy_term_reference',
- );
-
- return $field_bases;
-}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc
deleted file mode 100644
index 1f96e01..0000000
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.field_instance.inc
+++ /dev/null
@@ -1,91 +0,0 @@
-<?php
-/**
- * @file
- * taxonomy_opa_migrate.features.field_instance.inc
- */
-
-/**
- * Implements hook_field_default_field_instances().
- */
-function taxonomy_opa_migrate_field_default_field_instances() {
- $field_instances = array();
-
- // Exported field_instance: 'scald_atom-video-field_video_opa_genre'.
- $field_instances['scald_atom-video-field_video_opa_genre'] = array(
- 'bundle' => 'video',
- 'default_value' => NULL,
- 'deleted' => 0,
- 'description' => '',
- 'display' => array(
- 'default' => array(
- 'label' => 'above',
- 'module' => 'i18n_taxonomy',
- 'settings' => array(),
- 'type' => 'i18n_taxonomy_term_reference_link',
- 'weight' => 45,
- ),
- ),
- 'entity_type' => 'scald_atom',
- 'field_name' => 'field_video_opa_genre',
- 'label' => 'OPA Genre',
- 'required' => 0,
- 'settings' => array(
- 'entity_translation_sync' => FALSE,
- 'user_register_form' => FALSE,
- ),
- 'widget' => array(
- 'active' => 1,
- 'module' => 'options',
- 'settings' => array(),
- 'type' => 'options_buttons',
- 'weight' => 31,
- ),
- );
-
- // Exported field_instance:
- // 'taxonomy_term-foundation_opa_genres-field_opa_code'.
- $field_instances['taxonomy_term-foundation_opa_genres-field_opa_code'] = array(
- 'bundle' => 'foundation_opa_genres',
- 'default_value' => NULL,
- 'deleted' => 0,
- 'description' => '',
- 'display' => array(
- 'default' => array(
- 'label' => 'above',
- 'module' => 'text',
- 'settings' => array(),
- 'type' => 'text_default',
- 'weight' => 0,
- ),
- ),
- 'entity_type' => 'taxonomy_term',
- 'field_name' => 'field_opa_code',
- 'label' => 'Code',
- 'required' => 1,
- 'settings' => array(
- 'context' => 'title',
- 'context_default' => 'sdl_editor_representation',
- 'dnd_enabled' => 0,
- 'entity_translation_sync' => FALSE,
- 'mee_enabled' => 0,
- 'text_processing' => 0,
- 'user_register_form' => FALSE,
- ),
- 'widget' => array(
- 'active' => 1,
- 'module' => 'text',
- 'settings' => array(
- 'size' => 60,
- ),
- 'type' => 'text_textfield',
- 'weight' => 41,
- ),
- );
-
- // Translatables
- // Included for use with string extractors like potx.
- t('Code');
- t('OPA Genre');
-
- return $field_instances;
-}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.inc
deleted file mode 100644
index 9d44dae..0000000
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.inc
+++ /dev/null
@@ -1,14 +0,0 @@
-<?php
-/**
- * @file
- * taxonomy_opa_migrate.features.inc
- */
-
-/**
- * Implements hook_ctools_plugin_api().
- */
-function taxonomy_opa_migrate_ctools_plugin_api($module = NULL, $api = NULL) {
- if ($module == "strongarm" && $api == "strongarm") {
- return array("version" => "1");
- }
-}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.taxonomy.inc b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.taxonomy.inc
deleted file mode 100644
index 3b98ab1..0000000
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.features.taxonomy.inc
+++ /dev/null
@@ -1,38 +0,0 @@
-<?php
-/**
- * @file
- * taxonomy_opa_migrate.features.taxonomy.inc
- */
-
-/**
- * Implements hook_taxonomy_default_vocabularies().
- */
-function taxonomy_opa_migrate_taxonomy_default_vocabularies() {
- return array(
- 'foundation_opa_genres' => array(
- 'name' => 'Foundation Genres from OPA',
- 'machine_name' => 'foundation_opa_genres',
- 'description' => '',
- 'hierarchy' => 0,
- 'module' => 'taxonomy',
- 'weight' => 0,
- 'language' => 'und',
- 'i18n_mode' => 1,
- 'rdf_mapping' => array(
- 'rdftype' => array(
- 0 => 'skos:ConceptScheme',
- ),
- 'name' => array(
- 'predicates' => array(
- 0 => 'dc:title',
- ),
- ),
- 'description' => array(
- 'predicates' => array(
- 0 => 'rdfs:comment',
- ),
- ),
- ),
- ),
- );
-}
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
deleted file mode 100644
index ca6b8d4..0000000
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.info
+++ /dev/null
@@ -1,25 +0,0 @@
-name = Taxonomy OPA Migrate
-description = Provides Taxonomy Categories from OPA.
-core = 7.x
-package = ARTE Foundation
-dependencies[] = arte_opa
-dependencies[] = ctools
-dependencies[] = features
-dependencies[] = i18n_taxonomy
-dependencies[] = options
-dependencies[] = strongarm
-dependencies[] = taxonomy
-dependencies[] = text
-features[ctools][] = strongarm:strongarm:1
-features[features_api][] = api:2
-features[field_base][] = field_opa_code
-features[field_base][] = field_video_opa_genre
-features[field_instance][] = scald_atom-video-field_video_opa_genre
-features[field_instance][] = taxonomy_term-foundation_opa_genres-field_opa_code
-features[taxonomy][] = foundation_opa_genres
-; OPA Classes
-files[] = includes/opa_categories_reader.inc
-; Migrate source classes
-files[] = migrate/sources/taxonomy_opa.inc
-; Migrate classes
-files[] = migrate/taxonomy_opa_migrate.inc
diff --git a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module b/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
deleted file mode 100644
index 00e5d5f..0000000
--- a/modules/custom/taxonomy_opa_migrate/taxonomy_opa_migrate.module
+++ /dev/null
@@ -1,24 +0,0 @@
-<?php
-/**
- * @file
- * Code for the taxonomy_opa_migrate feature.
- */
-
-include_once 'taxonomy_opa_migrate.features.inc';
-
-/**
- * Implements hook_scald_opa_migrate_mapping_info().
- *
- * Provide specific mappings of OPA categories terms into Drupal taxonomies.
- *
- */
-function taxonomy_opa_migrate_scald_opa_migrate_mapping_info() {
- return array(
- 'opa_taxonomy' => array(
- array(
- 'field_name' => 'field_video_opa_genre',
- 'vocabulary_name' => 'foundation_opa_genres',
- ),
- ),
- );
-}
From c84e8433b24e169633b3cf7bd9675a4942e793cc Mon Sep 17 00:00:00 2001
From: lsantos-oq <luis.santos@openquest.pt>
Date: Fri, 17 Jun 2016 11:59:05 +0100
Subject: [PATCH 08/12] AFOUND-731 Invalidate atom to the next category
synchronization
---
.../categories_opa_migrate.features.field_base.inc | 29 +++++++++++
...egories_opa_migrate.features.field_instance.inc | 39 +++++++++++++++
.../categories_opa_migrate.info | 3 ++
.../categories_opa_migrate.module | 27 +++++++++++
.../migrate/categories_opa_migrate.inc | 46 ++++++++++++++++--
.../custom/scald_opa_migrate/migrate/scald_opa.inc | 56 ++++++----------------
6 files changed, 154 insertions(+), 46 deletions(-)
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc
index 378efa4..36197ba 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc
@@ -10,6 +10,35 @@
function categories_opa_migrate_field_default_field_bases() {
$field_bases = array();
+ // Exported field_base: 'field_categories_opa_update'.
+ $field_bases['field_categories_opa_update'] = array(
+ 'active' => 1,
+ 'cardinality' => 1,
+ 'deleted' => 0,
+ 'entity_types' => array(),
+ 'field_name' => 'field_categories_opa_update',
+ 'field_permissions' => array(
+ 'type' => 0,
+ ),
+ 'indexes' => array(
+ 'value' => array(
+ 0 => 'value',
+ ),
+ ),
+ 'locked' => 0,
+ 'module' => 'list',
+ 'settings' => array(
+ 'allowed_values' => array(
+ 0 => '',
+ 1 => '',
+ ),
+ 'allowed_values_function' => '',
+ 'entity_translation_sync' => FALSE,
+ ),
+ 'translatable' => 0,
+ 'type' => 'list_boolean',
+ );
+
// Exported field_base: 'field_opa_code'.
$field_bases['field_opa_code'] = array(
'active' => 1,
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
index 69a586d..92198c7 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
@@ -10,6 +10,44 @@
function categories_opa_migrate_field_default_field_instances() {
$field_instances = array();
+ // Exported field_instance: 'scald_atom-video-field_categories_opa_update'.
+ $field_instances['scald_atom-video-field_categories_opa_update'] = array(
+ 'bundle' => 'video',
+ 'default_value' => array(
+ 0 => array(
+ 'value' => 0,
+ ),
+ ),
+ 'deleted' => 0,
+ 'description' => '',
+ 'display' => array(
+ 'default' => array(
+ 'label' => 'above',
+ 'module' => 'list',
+ 'settings' => array(),
+ 'type' => 'list_default',
+ 'weight' => 46,
+ ),
+ ),
+ 'entity_type' => 'scald_atom',
+ 'field_name' => 'field_categories_opa_update',
+ 'label' => 'categories_opa_update',
+ 'required' => 0,
+ 'settings' => array(
+ 'entity_translation_sync' => FALSE,
+ 'user_register_form' => FALSE,
+ ),
+ 'widget' => array(
+ 'active' => 1,
+ 'module' => 'options',
+ 'settings' => array(
+ 'display_label' => 0,
+ ),
+ 'type' => 'options_onoff',
+ 'weight' => 32,
+ ),
+ );
+
// Exported field_instance: 'scald_atom-video-field_video_opa_genre'.
$field_instances['scald_atom-video-field_video_opa_genre'] = array(
'bundle' => 'video',
@@ -86,6 +124,7 @@ function categories_opa_migrate_field_default_field_instances() {
// Included for use with string extractors like potx.
t('Code');
t('OPA Genre');
+ t('categories_opa_update');
return $field_instances;
}
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.info b/modules/custom/categories_opa_migrate/categories_opa_migrate.info
index 1d7fe8b..f72557b 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.info
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.info
@@ -6,14 +6,17 @@ dependencies[] = arte_opa
dependencies[] = ctools
dependencies[] = features
dependencies[] = i18n_taxonomy
+dependencies[] = list
dependencies[] = options
dependencies[] = strongarm
dependencies[] = taxonomy
dependencies[] = text
features[ctools][] = strongarm:strongarm:1
features[features_api][] = api:2
+features[field_base][] = field_categories_opa_update
features[field_base][] = field_opa_code
features[field_base][] = field_video_opa_genre
+features[field_instance][] = scald_atom-video-field_categories_opa_update
features[field_instance][] = scald_atom-video-field_video_opa_genre
features[field_instance][] = taxonomy_term-foundation_opa_genres-field_opa_code
features[taxonomy][] = foundation_opa_genres
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.module b/modules/custom/categories_opa_migrate/categories_opa_migrate.module
index 814e299..035f87c 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.module
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.module
@@ -18,7 +18,34 @@ function categories_opa_migrate_scald_opa_migrate_mapping_info() {
array(
'field_name' => 'field_video_opa_genre',
'vocabulary_name' => 'foundation_opa_genres',
+ 'update_field' => 'field_categories_opa_update',
),
),
);
}
+
+/**
+ * Get tid from OPA Categories.
+ *
+ * @param [type] $vocabulary_name [description]
+ * @param [type] $categories [description]
+ */
+function categories_opa_migrate_fetch_terms($vocabulary_name, $categories) {
+ $tids = array();
+
+ foreach ($categories as $category) {
+ $query = new EntityFieldQuery();
+ $query = $query->entityCondition('entity_type', 'taxonomy_term')
+ ->entityCondition('bundle', $vocabulary_name)
+ ->fieldCondition('field_opa_code', 'value', $category['code'], '=');
+ $result = $query->execute();
+
+ if (isset($result['taxonomy_term'])) {
+ $terms = $result['taxonomy_term'];
+ $keys = array_keys($terms);
+ $tids = array_merge($tids, $keys);
+ }
+ }
+
+ return $tids;
+}
diff --git a/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc b/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
index 5e1e730..2392d3d 100644
--- a/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
+++ b/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
@@ -92,12 +92,12 @@ class CategoriesOPAMigration extends Migration {
$hook = isset($entity->original) ? 'update' : 'insert';
// Logging
- $msg = 'Taxonomy term %verb: %no_emiarte %kind, %title.';
+ $msg = 'Taxonomy term %verb: %code %kind, %title.';
$vars = array(
'%verb' => isset($entity->original) ? 'updated' : 'created',
- '%no_emiarte' => $entity->base_id,
+ '%code' => $row->code,
'%kind' => $row->kind,
- '%title' => $entity->title,
+ '%title' => $entity->name,
);
$link = l('Taxonomy term ' . $entity->tid, 'taxonomy/term/' . $entity->tid);
watchdog('categories_opa_migrate', $msg, $vars, WATCHDOG_DEBUG, $link);
@@ -111,7 +111,7 @@ class CategoriesOPAMigration extends Migration {
$i18n_category = new MigrateSourceCategoriesOPA($this->opa, $query);
while ($i18n_row = $i18n_category->getNextRow()) {
- $report = array();
+ $report = array('additions' => 0, 'deletes' => 0, 'updates' => 0, 'skips' => 0);
_locale_import_one_string_db($report, $i18n_row->language, 'term:'. $entity->tid .':name', $row->label, $i18n_row->label, 'Taxonomy', '', LOCALE_IMPORT_OVERWRITE);
}
@@ -123,6 +123,44 @@ class CategoriesOPAMigration extends Migration {
* Called when the import has finished
*/
public function postImport() {
+ // Check the atoms with flag to update category
+ $query = new EntityFieldQuery();
+ $query = $query->entityCondition('entity_type', 'scald_atom')
+ ->entityCondition('bundle', 'video')
+ ->fieldCondition('field_categories_opa_update', 'value', 1, '=');
+ $result = $query->execute();
+
+ if (isset($result['scald_atom'])) {
+ $atom_sids = array();
+ foreach ($result['scald_atom'] as $scald_atom) {
+ $atom_sids[] = $scald_atom->sid;
+ }
+
+ // Load Atoms
+ $atoms = entity_load('scald_atom', $atom_sids);
+ foreach ($atoms as $atom) {
+ // Get the first language
+
+ $json = reset($atom->data['json']);
+ $categories = categories_opa_migrate_fetch_terms('foundation_opa_genres', $json['categories']);
+
+ // Add the taxonomy
+ if (!empty($categories)) {
+ // Update the categories
+ $atom->field_video_opa_genre[LANGUAGE_NONE] = array();
+ foreach ($categories as $category) {
+ $atom->field_video_opa_genre[LANGUAGE_NONE][] = array('tid' => $category);
+ }
+
+ // Remove the flag
+ $atom->field_categories_opa_update[LANGUAGE_NONE][0]['value'] = 0;
+
+ //Save the atom
+ scald_atom_save($atom);
+ }
+ }
+ }
+
// Notify that the import has finished
module_invoke_all('categories_opa_migrate_complete', $this->processedTerms);
}
diff --git a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
index 3338e27..12aca5a 100644
--- a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
+++ b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
@@ -452,26 +452,24 @@ class ScaldOPAMigration extends Migration {
// Each mapping provides a taxonomy - thesaurus mapping.
foreach ($mappings as $mapping) {
if (!empty($row->categories)) {
- $categories_import = array();
-
- // Loop through all thesaurus terms and save the terms we should import
- foreach ($row->categories as $category) {
- $categories_import[] = $category;
- }
-
// Sync up terms, and take the returned tids
// We also make sure to include the parents
$key = $mapping['field_name'] . '_tids';
- $row->$key = $this->fetchCategories($mapping['vocabulary_name'], $categories_import);
- // Add the taxonomy mappings
- if (!empty($row->$key)) {
- $this->addFieldMapping($mapping['field_name'], $key, FALSE);
- $this->addFieldMapping($mapping['field_name'] . ':source_type', NULL, FALSE)
- ->defaultValue(TRUE);
- }
- else {
- $this->removeFieldMapping($mapping['field_name']);
+ if (function_exists('categories_opa_migrate_fetch_terms')) {
+ $row->$key = categories_opa_migrate_fetch_terms($mapping['vocabulary_name'], $row->categories);
+
+ // Add the taxonomy mappings
+ if (!empty($row->$key)) {
+ $this->addFieldMapping($mapping['field_name'], $key, FALSE);
+ $this->addFieldMapping($mapping['field_name'] . ':source_type', NULL, FALSE)
+ ->defaultValue(TRUE);
+ }
+ else {
+ // Flag this atom for update when the categories opa run
+ $this->addFieldMapping($mapping['update_field'], $key, FALSE);
+ $row->$mapping['update_field'] = 1;
+ }
}
}
}
@@ -479,32 +477,6 @@ class ScaldOPAMigration extends Migration {
}
/**
- * Get tid from OPA Categories.
- *
- * @param [type] $vocabulary_name [description]
- * @param [type] $categories [description]
- */
- protected function fetchCategories($vocabulary_name, $categories) {
- $tids = array();
-
- foreach ($categories as $category) {
- $query = new EntityFieldQuery();
- $query = $query->entityCondition('entity_type', 'taxonomy_term')
- ->entityCondition('bundle', $vocabulary_name)
- ->fieldCondition('field_opa_code', 'value', $category['code'], '=');
- $result = $query->execute();
-
- if (isset($result['taxonomy_term'])) {
- $terms = $result['taxonomy_term'];
- $keys = array_keys($terms);
- $tids = array_merge($tids, $keys);
- }
- }
-
- return $tids;
- }
-
- /**
* Helper function to sort terms comping from OPA. Used in PHP usort.
*
* @param [type] $a [description]
From 598ab86865bb97c298ab459f3d10f4c85394df6a Mon Sep 17 00:00:00 2001
From: lsantos-oq <luis.santos@openquest.pt>
Date: Fri, 17 Jun 2016 12:27:03 +0100
Subject: [PATCH 09/12] AFOUND-731 Fix value of category update field
---
.../categories_opa_migrate.features.field_instance.inc | 2 +-
modules/custom/scald_opa_migrate/migrate/scald_opa.inc | 3 +--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
index 92198c7..f93e54d 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
@@ -31,7 +31,7 @@ function categories_opa_migrate_field_default_field_instances() {
),
'entity_type' => 'scald_atom',
'field_name' => 'field_categories_opa_update',
- 'label' => 'categories_opa_update',
+ 'label' => 'Update OPA Categories',
'required' => 0,
'settings' => array(
'entity_translation_sync' => FALSE,
diff --git a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
index 12aca5a..653a722 100644
--- a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
+++ b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
@@ -467,8 +467,7 @@ class ScaldOPAMigration extends Migration {
}
else {
// Flag this atom for update when the categories opa run
- $this->addFieldMapping($mapping['update_field'], $key, FALSE);
- $row->$mapping['update_field'] = 1;
+ $this->addFieldMapping($mapping['update_field'], NULL, TRUE)->defaultValue(1);
}
}
}
From f3e95e153e8569d1743e268163904f3cd29fe94b Mon Sep 17 00:00:00 2001
From: Luis Correia <luis@marzeelabs.org>
Date: Mon, 20 Jun 2016 13:21:16 +0100
Subject: [PATCH 10/12] AFOUND-731 Rename fields and fix comments
---
modules/custom/categories_opa_migrate/README.md | 8 ++++----
.../categories_opa_migrate.drush.inc | 12 ++++++------
.../categories_opa_migrate.features.field_base.inc | 14 +++++++-------
...egories_opa_migrate.features.field_instance.inc | 22 +++++++++++-----------
.../categories_opa_migrate.features.taxonomy.inc | 6 +++---
.../categories_opa_migrate.info | 12 ++++++------
.../categories_opa_migrate.module | 6 +++---
.../migrate/categories_opa_migrate.inc | 10 +++++-----
8 files changed, 45 insertions(+), 45 deletions(-)
diff --git a/modules/custom/categories_opa_migrate/README.md b/modules/custom/categories_opa_migrate/README.md
index c35dd6c..9b943b4 100644
--- a/modules/custom/categories_opa_migrate/README.md
+++ b/modules/custom/categories_opa_migrate/README.md
@@ -3,7 +3,7 @@ Categories OPA Migrate
Module providing support for migrating from OPA Categories into `taxonomy terms`. Dynamic migrations use the migrate framework and Drush commands are supported to run migrations via the command-line.
-The Taxonomy that you select to put the migrate items need have the Translation mode: `Localize. Terms are common for all languages, but their name and description may be localized.`
+The Taxonomy that you select to put the migrate items into needs to have the Translation mode: `Localize. Terms are common for all languages, but their name and description may be localized.`
### Configuring OPA
@@ -23,8 +23,8 @@ The default configuration is querying OPA directly with the given parameters (se
The module provides a custom Drush command for synchronization
- drush cos
+ drush catopa
-See `drush help tos` for examples. For example see examples
+See `drush help catopa` for examples. For example see examples
- drush tos --platform=ALW
+ drush catopa --platform=ALW
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc
index 00f80cf..1b8421c 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc
@@ -13,16 +13,16 @@ function categories_opa_migrate_drush_command() {
$items['categories-opa-sync'] = array(
'description' => 'Import categories from OPA into Taxonomy terms',
- 'aliases' => array('cos'),
+ 'aliases' => array('catopa'),
'options' => array(
- 'platform' => 'The platform, e.g. ALW or AC. This is an obligatory option',
+ 'platform' => 'The platform, e.g. ALW or AC. This is a mandatory option',
'query' => 'E.g. "sort=language". Should have the format of URL GET parameters, and it might overwrite previously set parameters such as platform, code or language.',
'limit' => 'Limit on the length of each migration process, expressed in seconds or number of items',
'feedback' => 'Provides feedback during migration, see migration feedback option for more information',
'debug-opa' => 'Output debugging information about the OPA requests issued during synchronization',
),
'examples' => array(
- 'tos --platform=ALW --feedback="10 seconds"' => 'Imports the categories to the Foundation genres Taxonomy from the ALW platform having feedback every 10 seconds',
+ 'catopa --platform=ALW --feedback="10 seconds"' => 'Imports the categories to the Foundation OPA Categories Taxonomy for the ALW platform having feedback every 10 seconds',
),
);
@@ -33,7 +33,7 @@ function categories_opa_migrate_drush_command() {
* Command callback: run Categories OPA synchronization
*/
function drush_categories_opa_migrate_categories_opa_sync($args = NULL) {
- // Fetch the platform - this option is obligatory and will fail if not provided
+ // Fetch the platform - this option is mandatory and will fail if not provided
$platform = drush_get_option('platform', FALSE);
if (!$platform || $platform == 1) {
drush_log(dt('Missing or empty parameter --platform'), 'error');
@@ -51,11 +51,11 @@ function drush_categories_opa_migrate_categories_opa_sync($args = NULL) {
$arguments['query'] = array(
'platform' => drush_get_option('platform'),
'kind' => 'INTERNET',
- 'language' => variable_get('categories_opa_migrate_base_language', 'en'),
+ 'language' => variable_get('categories_opa_migrate_base_language', 'fr'),
);
// Set the taxonomy bundle
- $arguments['bundle'] = 'foundation_opa_genres';
+ $arguments['bundle'] = 'foundation_opa_categories';
// Additional query parameters
if ($query_additional = drush_get_option('query', FALSE)) {
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc
index 36197ba..ad89d4c 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_base.inc
@@ -10,13 +10,13 @@
function categories_opa_migrate_field_default_field_bases() {
$field_bases = array();
- // Exported field_base: 'field_categories_opa_update'.
- $field_bases['field_categories_opa_update'] = array(
+ // Exported field_base: 'field_video_opa_categories_pending'.
+ $field_bases['field_video_opa_categories_pending'] = array(
'active' => 1,
'cardinality' => 1,
'deleted' => 0,
'entity_types' => array(),
- 'field_name' => 'field_categories_opa_update',
+ 'field_name' => 'field_video_opa_categories_pending',
'field_permissions' => array(
'type' => 0,
),
@@ -64,13 +64,13 @@ function categories_opa_migrate_field_default_field_bases() {
'type' => 'text',
);
- // Exported field_base: 'field_video_opa_genre'.
- $field_bases['field_video_opa_genre'] = array(
+ // Exported field_base: 'field_video_opa_categories'.
+ $field_bases['field_video_opa_categories'] = array(
'active' => 1,
'cardinality' => -1,
'deleted' => 0,
'entity_types' => array(),
- 'field_name' => 'field_video_opa_genre',
+ 'field_name' => 'field_video_opa_categories',
'field_permissions' => array(
'type' => 0,
),
@@ -84,7 +84,7 @@ function categories_opa_migrate_field_default_field_bases() {
'settings' => array(
'allowed_values' => array(
0 => array(
- 'vocabulary' => 'foundation_opa_genres',
+ 'vocabulary' => 'foundation_opa_categories',
'parent' => 0,
),
),
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
index f93e54d..a0e3afa 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.field_instance.inc
@@ -10,8 +10,8 @@
function categories_opa_migrate_field_default_field_instances() {
$field_instances = array();
- // Exported field_instance: 'scald_atom-video-field_categories_opa_update'.
- $field_instances['scald_atom-video-field_categories_opa_update'] = array(
+ // Exported field_instance: 'scald_atom-video-field_video_opa_categories_pending'.
+ $field_instances['scald_atom-video-field_video_opa_categories_pending'] = array(
'bundle' => 'video',
'default_value' => array(
0 => array(
@@ -30,7 +30,7 @@ function categories_opa_migrate_field_default_field_instances() {
),
),
'entity_type' => 'scald_atom',
- 'field_name' => 'field_categories_opa_update',
+ 'field_name' => 'field_video_opa_categories_pending',
'label' => 'Update OPA Categories',
'required' => 0,
'settings' => array(
@@ -48,8 +48,8 @@ function categories_opa_migrate_field_default_field_instances() {
),
);
- // Exported field_instance: 'scald_atom-video-field_video_opa_genre'.
- $field_instances['scald_atom-video-field_video_opa_genre'] = array(
+ // Exported field_instance: 'scald_atom-video-field_video_opa_categories'.
+ $field_instances['scald_atom-video-field_video_opa_categories'] = array(
'bundle' => 'video',
'default_value' => NULL,
'deleted' => 0,
@@ -64,8 +64,8 @@ function categories_opa_migrate_field_default_field_instances() {
),
),
'entity_type' => 'scald_atom',
- 'field_name' => 'field_video_opa_genre',
- 'label' => 'OPA Genre',
+ 'field_name' => 'field_video_opa_categories',
+ 'label' => 'OPA Categories',
'required' => 0,
'settings' => array(
'entity_translation_sync' => FALSE,
@@ -81,9 +81,9 @@ function categories_opa_migrate_field_default_field_instances() {
);
// Exported field_instance:
- // 'taxonomy_term-foundation_opa_genres-field_opa_code'.
- $field_instances['taxonomy_term-foundation_opa_genres-field_opa_code'] = array(
- 'bundle' => 'foundation_opa_genres',
+ // 'taxonomy_term-foundation_opa_categories-field_opa_code'.
+ $field_instances['taxonomy_term-foundation_opa_categories-field_opa_code'] = array(
+ 'bundle' => 'foundation_opa_categories',
'default_value' => NULL,
'deleted' => 0,
'description' => '',
@@ -123,7 +123,7 @@ function categories_opa_migrate_field_default_field_instances() {
// Translatables
// Included for use with string extractors like potx.
t('Code');
- t('OPA Genre');
+ t('OPA Categories');
t('categories_opa_update');
return $field_instances;
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.taxonomy.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.taxonomy.inc
index bdfb6a0..041af72 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.features.taxonomy.inc
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.features.taxonomy.inc
@@ -9,9 +9,9 @@
*/
function categories_opa_migrate_taxonomy_default_vocabularies() {
return array(
- 'foundation_opa_genres' => array(
- 'name' => 'Foundation Genres from OPA',
- 'machine_name' => 'foundation_opa_genres',
+ 'foundation_opa_categories' => array(
+ 'name' => 'Foundation Categories from OPA',
+ 'machine_name' => 'foundation_opa_categories',
'description' => '',
'hierarchy' => 0,
'module' => 'taxonomy',
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.info b/modules/custom/categories_opa_migrate/categories_opa_migrate.info
index f72557b..089f4e1 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.info
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.info
@@ -13,13 +13,13 @@ dependencies[] = taxonomy
dependencies[] = text
features[ctools][] = strongarm:strongarm:1
features[features_api][] = api:2
-features[field_base][] = field_categories_opa_update
+features[field_base][] = field_video_opa_categories_pending
features[field_base][] = field_opa_code
-features[field_base][] = field_video_opa_genre
-features[field_instance][] = scald_atom-video-field_categories_opa_update
-features[field_instance][] = scald_atom-video-field_video_opa_genre
-features[field_instance][] = taxonomy_term-foundation_opa_genres-field_opa_code
-features[taxonomy][] = foundation_opa_genres
+features[field_base][] = field_video_opa_categories
+features[field_instance][] = scald_atom-video-field_video_opa_categories_pending
+features[field_instance][] = scald_atom-video-field_video_opa_categories
+features[field_instance][] = taxonomy_term-foundation_opa_categories-field_opa_code
+features[taxonomy][] = foundation_opa_categories
; OPA Classes
files[] = includes/opa_categories_reader.inc
; Migrate source classes
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.module b/modules/custom/categories_opa_migrate/categories_opa_migrate.module
index 035f87c..7733beb 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.module
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.module
@@ -16,9 +16,9 @@ function categories_opa_migrate_scald_opa_migrate_mapping_info() {
return array(
'opa_categories' => array(
array(
- 'field_name' => 'field_video_opa_genre',
- 'vocabulary_name' => 'foundation_opa_genres',
- 'update_field' => 'field_categories_opa_update',
+ 'field_name' => 'field_video_opa_categories',
+ 'vocabulary_name' => 'foundation_opa_categories',
+ 'update_field' => 'field_video_opa_categories_pending',
),
),
);
diff --git a/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc b/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
index 2392d3d..f314d6f 100644
--- a/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
+++ b/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
@@ -127,7 +127,7 @@ class CategoriesOPAMigration extends Migration {
$query = new EntityFieldQuery();
$query = $query->entityCondition('entity_type', 'scald_atom')
->entityCondition('bundle', 'video')
- ->fieldCondition('field_categories_opa_update', 'value', 1, '=');
+ ->fieldCondition('field_video_opa_categories_pending', 'value', 1, '=');
$result = $query->execute();
if (isset($result['scald_atom'])) {
@@ -142,18 +142,18 @@ class CategoriesOPAMigration extends Migration {
// Get the first language
$json = reset($atom->data['json']);
- $categories = categories_opa_migrate_fetch_terms('foundation_opa_genres', $json['categories']);
+ $categories = categories_opa_migrate_fetch_terms('foundation_opa_categories', $json['categories']);
// Add the taxonomy
if (!empty($categories)) {
// Update the categories
- $atom->field_video_opa_genre[LANGUAGE_NONE] = array();
+ $atom->field_video_opa_categories[LANGUAGE_NONE] = array();
foreach ($categories as $category) {
- $atom->field_video_opa_genre[LANGUAGE_NONE][] = array('tid' => $category);
+ $atom->field_video_opa_categories[LANGUAGE_NONE][] = array('tid' => $category);
}
// Remove the flag
- $atom->field_categories_opa_update[LANGUAGE_NONE][0]['value'] = 0;
+ $atom->field_video_opa_categories_pending[LANGUAGE_NONE][0]['value'] = 0;
//Save the atom
scald_atom_save($atom);
From 5fa08b7503d9c3e66373ee77bd8ebc138c3b3852 Mon Sep 17 00:00:00 2001
From: Luis Correia <luis@marzeelabs.org>
Date: Mon, 20 Jun 2016 14:25:56 +0100
Subject: [PATCH 11/12] AFOUND-731 Bring code up to Drupal standards
---
.../categories_opa_migrate.drush.inc | 28 ++++----
.../includes/opa_categories_reader.inc | 36 +++++-----
.../migrate/categories_opa_migrate.inc | 77 +++++++++++++---------
.../migrate/sources/categories_opa.inc | 29 ++++----
4 files changed, 96 insertions(+), 74 deletions(-)
diff --git a/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc b/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc
index 1b8421c..278cf2b 100644
--- a/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc
+++ b/modules/custom/categories_opa_migrate/categories_opa_migrate.drush.inc
@@ -1,8 +1,7 @@
<?php
/**
* @file
- *
- * Provides drush extensions for Categories OPA
+ * Provides drush extensions for Categories OPA.
*/
/**
@@ -30,10 +29,10 @@ function categories_opa_migrate_drush_command() {
}
/**
- * Command callback: run Categories OPA synchronization
+ * Command callback: run Categories OPA synchronization.
*/
function drush_categories_opa_migrate_categories_opa_sync($args = NULL) {
- // Fetch the platform - this option is mandatory and will fail if not provided
+ // Fetch the platform - this option is mandatory and will fail when provided.
$platform = drush_get_option('platform', FALSE);
if (!$platform || $platform == 1) {
drush_log(dt('Missing or empty parameter --platform'), 'error');
@@ -41,23 +40,22 @@ function drush_categories_opa_migrate_categories_opa_sync($args = NULL) {
}
$arguments = array(
- //'needsUpdate' => drush_get_option('update', FALSE),
+ // 'needsUpdate' => drush_get_option('update', FALSE),
'debug' => drush_get_option('debug-opa', FALSE),
- //'uniqueEmNumber' => drush_get_option('uniqueEmNumber', FALSE),
+ // 'uniqueEmNumber' => drush_get_option('uniqueEmNumber', FALSE),
);
-
- // Default query arguments
+ // Default query arguments.
$arguments['query'] = array(
'platform' => drush_get_option('platform'),
'kind' => 'INTERNET',
'language' => variable_get('categories_opa_migrate_base_language', 'fr'),
);
- // Set the taxonomy bundle
+ // Set the taxonomy bundle.
$arguments['bundle'] = 'foundation_opa_categories';
- // Additional query parameters
+ // Additional query parameters.
if ($query_additional = drush_get_option('query', FALSE)) {
$parts = explode('&', $query_additional);
if (!empty($parts)) {
@@ -70,13 +68,13 @@ function drush_categories_opa_migrate_categories_opa_sync($args = NULL) {
}
}
- // Limit
+ // Limit.
if ($limit = drush_get_option('limit')) {
$options['limit']['value'] = $limit;
$options['limit']['unit'] = 'items';
}
- // Feedback
+ // Feedback.
$feedback = drush_get_option('feedback');
if ($feedback) {
$parts = explode(' ', $feedback);
@@ -92,12 +90,12 @@ function drush_categories_opa_migrate_categories_opa_sync($args = NULL) {
}
}
- // Run the migration
+ // Run the migration.
_drush_categories_opa_migrate_run_migrate($arguments, $options);
}
/**
- *
+ * Auxiliary function to start the migation.
*/
function _drush_categories_opa_migrate_run_migrate($arguments = array(), $options = array()) {
$machine_name = 'CategoriesOPA';
@@ -116,7 +114,7 @@ function _drush_categories_opa_migrate_run_migrate($arguments = array(), $option
$status = $migration->processImport($options);
// Reset all caches, if not recreating our migration class with different
- // parameters won't make any difference
+ // parameters won't make any difference.
drupal_static_reset();
// Clear cache and force refresh of JavaScript translations.
diff --git a/modules/custom/categories_opa_migrate/includes/opa_categories_reader.inc b/modules/custom/categories_opa_migrate/includes/opa_categories_reader.inc
index 67f079a..4daad8f 100644
--- a/modules/custom/categories_opa_migrate/includes/opa_categories_reader.inc
+++ b/modules/custom/categories_opa_migrate/includes/opa_categories_reader.inc
@@ -1,10 +1,12 @@
<?php
-
/**
* @file
- * OPA Categories reader
+ * OPA Categories reader.
*/
+/**
+ * Class OPACategoriesReader.
+ */
class OPACategoriesReader extends OPAReader {
/**
@@ -21,37 +23,37 @@ class OPACategoriesReader extends OPAReader {
* object's unique identifier) from the specified JSON file. Sets both to
* NULL at end-of-file. Handles properly-formed JSON, as well as some improper
* coding (specifically that generated in Ning exports).
- *
- * @return void
*/
public function next() {
- // Use pagination when no user limit was specified or when the specified limit is
- // bigger than the maximum allowed by OPA
+ // Use pagination when no user limit was specified or when the specified
+ // limit is bigger than the maximum allowed by OPA.
$pagination = is_null($this->limit) || $this->limit > $this->pagination_limit;
- // Open the file and position it if necessary
+ // Open the file and position it if necessary.
if (empty($this->data)) {
$options = $this->params;
$options['page'] = $this->page;
$options['limit'] = $pagination ? $this->pagination_limit : $this->limit;
- // API method callback with pagination and limits
+ // API method callback with pagination and limits.
$this->data = $this->opa->apiCall($this->resourcePath(), $options);
- // @todo add try/catch
+ // @todo add try/catch.
if (!$this->data) {
return FALSE;
}
}
- // Increment Id
+ // Increment Id.
$this->position++;
- // If we're paginating with a user-specified limit, check if the iterator has reached the limit
+ // If we're paginating with a user-specified limit,
+ // check if the iterator has reached the limit.
//
- // NB the last OPA page will contain the pagination limit, i.e. 50 items by default.
- // If however the user-specified limit is 60 for instance, we only want the remaining 10 items from
- // that last page.
+ // NB the last OPA page will contain the pagination limit, i.e.,
+ // 50 items by default.
+ // If however the user-specified limit is 60 for instance,
+ // we only want the remaining 10 items from that last page.
$last_page_delta = $this->limit - (($this->page - 1) * $this->pagination_limit);
if (!is_null($this->limit) && $pagination && $this->position + 1 > $last_page_delta) {
// We are done!
@@ -62,9 +64,10 @@ class OPACategoriesReader extends OPAReader {
$this->currentId = $this->currentElement['code'];
}
else {
- // Check if we need to advance the pagination
+ // Check if we need to advance the pagination.
if (isset($this->data['meta'][$this->resource]['pages']) && $pagination && $this->page < $this->data['meta'][$this->resource]['pages']) {
- // Yes, and we have pages available still, so increment page and reset data to fetch the next page
+ // Yes, and we have pages available still,
+ // so increment page and reset data to fetch the next page.
$this->page++;
$this->position = -1;
$this->data = array();
@@ -76,4 +79,5 @@ class OPACategoriesReader extends OPAReader {
}
}
}
+
}
diff --git a/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc b/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
index f314d6f..055ad37 100644
--- a/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
+++ b/modules/custom/categories_opa_migrate/migrate/categories_opa_migrate.inc
@@ -1,34 +1,40 @@
<?php
-
/**
* @file
* Migration class for Categories OPA migrations.
*/
+/**
+ * Class CategoriesOPAMigration.
+ */
class CategoriesOPAMigration extends Migration {
protected $key = 'code';
- // Custom mappings as provided by modules implementing hook_categories_opa_migrate_mapping_info();
+ // Custom mappings as provided by modules
+ // implementing hook_categories_opa_migrate_mapping_info();
protected $customMappings = array();
- // OPA connection
- protected $opa = null;
+ // OPA connection.
+ protected $opa = NULL;
- // The translation query
+ // The translation query.
protected $translationsQuery = array();
- // Internal store of newly created or updated terms objects
+ // Internal store of newly created or updated terms objects.
protected $processedTerms = array();
+ /**
+ * Constructor.
+ */
public function __construct($arguments) {
parent::__construct($arguments);
$this->description = t('Import categories into Taxonomy terms.');
// Create a map object for tracking the relationships between source rows.
- // If we want to enforce unique emission numbers also outside of this migration,
- // we will use the MigrateUniqueEMNumberSQLMap class instead of the default
- // MigrateSQLMap class
+ // If we want to enforce unique emission numbers also outside of this
+ // migration, we will use the MigrateUniqueEMNumberSQLMap class instead of
+ // the default MigrateSQLMap class.
$mapClass = 'MigrateSQLMap';
if (isset($arguments['uniqueEmNumber']) && $arguments['uniqueEmNumber']) {
$mapClass = 'MigrateUniqueEMNumberSQLMap';
@@ -45,10 +51,10 @@ class CategoriesOPAMigration extends Migration {
MigrateDestinationTerm::getKeySchema()
);
- // Destination
+ // Destination.
$this->destination = new MigrateDestinationTerm($arguments['bundle']);
- // Get an OPA connection
+ // Get an OPA connection.
$this->opa = arte_opa_get_api();
// Use debug configuration for OPA?
@@ -56,11 +62,11 @@ class CategoriesOPAMigration extends Migration {
$opa->setDebug();
}
- // Source: OPA with parameters
+ // Source: OPA with parameters.
$query = isset($arguments['query']) ? $arguments['query'] : array();
$this->source = new MigrateSourceCategoriesOPA($this->opa, $query);
- // Build the translation query
+ // Build the translation query.
$this->translationsQuery = isset($arguments['query']) ? $arguments['query'] : array();
$languages_enabled = arte_opa_languages();
@@ -69,7 +75,7 @@ class CategoriesOPAMigration extends Migration {
}
$this->translationsQuery['language'] = implode(',', $languages_enabled);
- // Field mappings
+ // Field mappings.
$this->addFieldMapping('name', 'label');
$this->addFieldMapping('weight', 'order');
$this->addFieldMapping('field_opa_code', 'code');
@@ -78,6 +84,9 @@ class CategoriesOPAMigration extends Migration {
$this->customMappings = module_invoke_all('categories_opa_migrate_mapping_info');
}
+ /**
+ * Prepare row.
+ */
public function prepareRow($row) {
if (parent::prepareRow($row) === FALSE) {
return FALSE;
@@ -88,10 +97,11 @@ class CategoriesOPAMigration extends Migration {
* Called immediately after the complete destination object is saved.
*/
public function complete($entity, $row) {
- // Find out if this was a new taxonomy term or the updated of an existing one
+ // Find out if this was a new taxonomy term or
+ // the updated of an existing one.
$hook = isset($entity->original) ? 'update' : 'insert';
- // Logging
+ // Logging.
$msg = 'Taxonomy term %verb: %code %kind, %title.';
$vars = array(
'%verb' => isset($entity->original) ? 'updated' : 'created',
@@ -99,20 +109,25 @@ class CategoriesOPAMigration extends Migration {
'%kind' => $row->kind,
'%title' => $entity->name,
);
- $link = l('Taxonomy term ' . $entity->tid, 'taxonomy/term/' . $entity->tid);
+ $link = l(t('Taxonomy term @entity_tid taxonomy/term/@entity_tid'), array('@entity_tid' => $entity->tid));
watchdog('categories_opa_migrate', $msg, $vars, WATCHDOG_DEBUG, $link);
- // Store processed atoms
+ // Store processed atoms.
$this->processedTerms[] = $entity;
- // Get the translations
+ // Get the translations.
$query = $this->translationsQuery;
$query['code'] = $row->code;
$i18n_category = new MigrateSourceCategoriesOPA($this->opa, $query);
while ($i18n_row = $i18n_category->getNextRow()) {
- $report = array('additions' => 0, 'deletes' => 0, 'updates' => 0, 'skips' => 0);
- _locale_import_one_string_db($report, $i18n_row->language, 'term:'. $entity->tid .':name', $row->label, $i18n_row->label, 'Taxonomy', '', LOCALE_IMPORT_OVERWRITE);
+ $report = array(
+ 'additions' => 0,
+ 'deletes' => 0,
+ 'updates' => 0,
+ 'skips' => 0,
+ );
+ _locale_import_one_string_db($report, $i18n_row->language, 'term:' . $entity->tid . ':name', $row->label, $i18n_row->label, 'Taxonomy', '', LOCALE_IMPORT_OVERWRITE);
}
// Notify that we have a new or updated scald atom.
@@ -120,10 +135,10 @@ class CategoriesOPAMigration extends Migration {
}
/**
- * Called when the import has finished
+ * Called when the import has finished.
*/
public function postImport() {
- // Check the atoms with flag to update category
+ // Check the atoms with flag to update category.
$query = new EntityFieldQuery();
$query = $query->entityCondition('entity_type', 'scald_atom')
->entityCondition('bundle', 'video')
@@ -136,32 +151,32 @@ class CategoriesOPAMigration extends Migration {
$atom_sids[] = $scald_atom->sid;
}
- // Load Atoms
+ // Load Atoms.
$atoms = entity_load('scald_atom', $atom_sids);
foreach ($atoms as $atom) {
- // Get the first language
-
+ // Get the first language.
$json = reset($atom->data['json']);
$categories = categories_opa_migrate_fetch_terms('foundation_opa_categories', $json['categories']);
- // Add the taxonomy
+ // Add the taxonomy.
if (!empty($categories)) {
- // Update the categories
+ // Update the categories.
$atom->field_video_opa_categories[LANGUAGE_NONE] = array();
foreach ($categories as $category) {
$atom->field_video_opa_categories[LANGUAGE_NONE][] = array('tid' => $category);
}
- // Remove the flag
+ // Remove the flag.
$atom->field_video_opa_categories_pending[LANGUAGE_NONE][0]['value'] = 0;
- //Save the atom
+ // Save the atom.
scald_atom_save($atom);
}
}
}
- // Notify that the import has finished
+ // Notify that the import has finished.
module_invoke_all('categories_opa_migrate_complete', $this->processedTerms);
}
+
}
diff --git a/modules/custom/categories_opa_migrate/migrate/sources/categories_opa.inc b/modules/custom/categories_opa_migrate/migrate/sources/categories_opa.inc
index 3529bba..4ddf974 100644
--- a/modules/custom/categories_opa_migrate/migrate/sources/categories_opa.inc
+++ b/modules/custom/categories_opa_migrate/migrate/sources/categories_opa.inc
@@ -1,10 +1,15 @@
<?php
-
/**
* @file
+ * OPA Categories Migration.
+ *
* Support for the OPA source, accepting an OPA class and parameters to make
* connections.
*/
+
+/**
+ * OPA Categories Migration.
+ */
class MigrateSourceCategoriesOPA extends MigrateSource {
protected $reader;
@@ -17,12 +22,12 @@ class MigrateSourceCategoriesOPA extends MigrateSource {
$this->fields = $fields;
- // Setup our reader
+ // Setup our reader.
$this->reader = new OPACategoriesReader($opa, $params);
}
/**
- * Return a string representing OPA
+ * Return a string representing OPA.
*/
public function __toString() {
return (string) $this->reader;
@@ -32,8 +37,8 @@ class MigrateSourceCategoriesOPA extends MigrateSource {
* Returns a list of fields available to be mapped from the source query.
*
* @return array
- * Keys: machine names of the fields (to be passed to addFieldMapping)
- * Values: Human-friendly descriptions of the fields.
+ * Keys: machine names of the fields (to be passed to addFieldMapping)
+ * Values: Human-friendly descriptions of the fields.
*/
public function fields() {
return $this->fields;
@@ -61,8 +66,8 @@ class MigrateSourceCategoriesOPA extends MigrateSource {
/**
* Implementation of MigrationSource::getNextRow().
*
- * @return stdClass
- * data for the next row from the JSON source files
+ * @return mixed
+ * stdClass data for the next row from the JSON source files
*/
public function getNextRow() {
migrate_instrument_start('MigrateSourceOPA::next');
@@ -70,7 +75,7 @@ class MigrateSourceCategoriesOPA extends MigrateSource {
$next = NULL;
if (!isset($next) && $this->reader->valid()) {
- $next = (object)$this->reader->current();
+ $next = (object) $this->reader->current();
}
if (!isset($next)) {
@@ -82,21 +87,21 @@ class MigrateSourceCategoriesOPA extends MigrateSource {
$this->reader->next();
if ($this->reader->valid()) {
- $next = (object)$this->reader->current();
+ $next = (object) $this->reader->current();
// Keep looping on our results if $row and $next have the
- // same primary key
+ // same primary key.
$same = ($row->code == $next->code && $row->kind == $next->kind && $row->language == $next->language);
}
else {
$next = NULL;
$same = FALSE;
}
- }
- while ($same && $next);
+ } while ($same && $next);
migrate_instrument_stop('MigrateSourceOPA::next');
return $row;
}
+
}
From 8ef404e665380908745ba8daa2ed807e6ad7c053 Mon Sep 17 00:00:00 2001
From: Luis Correia <luis@marzeelabs.org>
Date: Mon, 20 Jun 2016 14:57:14 +0100
Subject: [PATCH 12/12] AFOUND-731 Bring some code up to Drupal standards
---
.../custom/scald_opa_migrate/migrate/scald_opa.inc | 217 ++++++++++++---------
1 file changed, 127 insertions(+), 90 deletions(-)
diff --git a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
index 653a722..6b5f8b1 100644
--- a/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
+++ b/modules/custom/scald_opa_migrate/migrate/scald_opa.inc
@@ -5,41 +5,47 @@
* Migration class for Scald OPA atom migrations.
*/
+/**
+ * Scald OPA Migration.
+ */
class ScaldOPAMigration extends Migration {
// The main key for mapping and determining video uniqueness.
// See also prepareKey() for preparing that key from the source.
// @see http://jira.newtech.arte.tv/browse/AFOUND-102
protected $key = 'programId_kind';
- // The default language for any migration, can be used to set the main language
- // of e.g. taxonomy terms
+ // The default language for any migration, can be used to set the main
+ // language of e.g. taxonomy terms.
protected $defaultLanguage = 'fr';
- // Custom mappings as provided by modules implementing hook_scald_opa_migrate_mapping_info();
+ // Custom mappings as provided by modules implementing
+ // hook_scald_opa_migrate_mapping_info();
protected $customMappings = array();
- // Internal store of newly created or updated atom entity objects
+ // Internal store of newly created or updated atom entity objects.
protected $processedAtoms = array();
- // The name of the module that requested the import
+ // The name of the module that requested the import.
protected $module = NULL;
+ /**
+ * Constructor.
+ */
public function __construct($arguments) {
parent::__construct($arguments);
$this->description = t('Import OPA videos into scald atoms.');
-
// Optionally store the name of the module that requested the import in the
- // atom's data array
+ // atom's data array.
if (!empty($arguments['module'])) {
$this->module = $arguments['module'];
}
// Create a map object for tracking the relationships between source rows.
- // If we want to enforce unique emission numbers also outside of this migration,
- // we will use the MigrateUniqueEMNumberSQLMap class instead of the default
- // MigrateSQLMap class
+ // If we want to enforce unique emission numbers also outside of this
+ // migration, we will use the MigrateUniqueEMNumberSQLMap class instead of
+ // the default MigrateSQLMap class.
$mapClass = 'MigrateSQLMap';
if (isset($arguments['uniqueEmNumber']) && $arguments['uniqueEmNumber']) {
$mapClass = 'MigrateUniqueEMNumberSQLMap';
@@ -55,13 +61,13 @@ class ScaldOPAMigration extends Migration {
MigrateDestinationScaldAtomOPA::getKeySchema()
);
- // Destination
+ // Destination.
$needsUpdate = isset($arguments['needsUpdate']) ? $arguments['needsUpdate'] : FALSE;
$this->destination = new MigrateDestinationScaldAtomOPA($needsUpdate);
- // Source: test OPA data
+ // Source: test OPA data.
if (isset($arguments['test']) && $arguments['test']) {
- // Our test source: a list containing local JSONs from OPA
+ // Our test source: a list containing local JSONs from OPA.
$json_dir = variable_get('scald_opa_migrate_test_json_dir', drupal_get_path('module', 'scald_opa_migrate') . '/tests/json/');
$json_list = $json_dir . '/' . variable_get('scald_opa_migrate_test_json', 'list.json');
if (isset($arguments['rabbitmq']) && $arguments['rabbitmq']) {
@@ -71,9 +77,9 @@ class ScaldOPAMigration extends Migration {
$this->source = new MigrateSourceOPATest($json_list, $json_dir);
}
}
- // Source: OPA
+ // Source: OPA.
else {
- // Get an OPA connection
+ // Get an OPA connection.
$opa = arte_opa_get_api();
// Use debug configuration for OPA?
@@ -81,9 +87,9 @@ class ScaldOPAMigration extends Migration {
$opa->setDebug();
}
- // Combine with Rabbit
+ // Combine with Rabbit.
if (isset($arguments['rabbitmq']) && $arguments['rabbitmq']) {
- // Provide default languages
+ // Provide default languages.
if (!isset($arguments['languages'])) {
$arguments['languages'] = arte_opa_languages();
}
@@ -102,9 +108,9 @@ class ScaldOPAMigration extends Migration {
$arguments['languages']
);
}
- // Query OPA for the list
+ // Query OPA for the list.
else {
- // Source: OPA with parameters
+ // Source: OPA with parameters.
$query = isset($arguments['query']) ? $arguments['query'] : array();
$this->source = new MigrateSourceOPA(
$opa,
@@ -113,12 +119,11 @@ class ScaldOPAMigration extends Migration {
}
}
- // Field mappings
-
+ // Field mappings.
$this->addFieldMapping('type')->defaultValue('video');
$this->addFieldMapping('provider')->defaultValue('scald_opa');
- // All migrated atoms are owned by user 1
+ // All migrated atoms are owned by user 1.
$this->addFieldMapping('publisher')->defaultValue(1);
// The language of the fields.
@@ -133,7 +138,6 @@ class ScaldOPAMigration extends Migration {
$this->addFieldMapping('scald_thumbnail:file_class', NULL, FALSE)
->defaultValue('MigrateFileFid');
// $this->addFieldMapping('scald_thumbnail:title',
-
$this->addFieldMapping('field_video_start_date', 'videoRightsBegin');
$this->addFieldMapping('field_video_rights_until', 'videoRightsEnd');
$this->addFieldMapping('field_alw_recording_date', 'shootingDate');
@@ -146,13 +150,15 @@ class ScaldOPAMigration extends Migration {
// Grab custom mappings provided by other modules.
$this->customMappings = module_invoke_all('scald_opa_migrate_mapping_info');
- // Makes sure the thumbnail destination folder exists before processing migration.
- // Directory in which to save the images
+ // Makes sure the thumbnail destination folder exists before processing
+ // migration.
+ // Directory in which to save the images.
$file_directory = 'public://atoms/image/opa';
- // Make sure that directory exists, or create it if not
+ // Make sure that directory exists, or create it if not.
if (!file_prepare_directory($file_directory)) {
- // If the folder could not be created we throw an error and stop the process.
+ // If the folder could not be created we throw an error and stop the
+ // process.
if (!drupal_mkdir($file_directory, NULL, TRUE)) {
throw new MigrateException(t("Images destination folder !destfolder does not exist and could not be created. Please check permissions.",
array('!destfolder' => $file_directory)));
@@ -160,6 +166,9 @@ class ScaldOPAMigration extends Migration {
}
}
+ /**
+ * Prepare row.
+ */
public function prepareRow($row) {
if (parent::prepareRow($row) === FALSE) {
return FALSE;
@@ -169,38 +178,43 @@ class ScaldOPAMigration extends Migration {
// to use MigrateFileBase instead, will be much cleaner
$this->parseThumbnail($row);
- // Parse dates, specifically to make sure we're not hitting the Y2038 bug
+ // Parse dates, specifically to make sure we're not hitting the Y2038 bug.
$this->parseDates($row);
- // Parse thesaurus logic. This maps thesaurus terms on Drupal taxonomy terms
+ // Parse thesaurus logic. This maps thesaurus terms on Drupal taxonomy
+ // terms.
$this->parseThesaurus($row);
- // Parse categories logic. This maps categories terms on Drupal taxonomy terms
+ // Parse categories logic. This maps categories terms on Drupal taxonomy
+ // terms.
$this->parseCategories($row);
}
/**
* Give handlers a shot at modifying the object before saving it.
*
- * @param $entity
- * Entity object to build. Prefilled with any fields mapped in the Migration.
- * @param $source_row
- * Raw source data object - passed through to prepare handlers.
+ * @param object $entity
+ * Entity object to build. Prefilled with any fields mapped in the
+ * Migration.
+ * @param object $row
+ * Raw source data object - passed through to prepare handlers.
*/
function prepare($entity, $row) {
- // Add the JSON dump as well to the entity. Deep-convert to array beforehand.
+ // Add the JSON dump as well to the entity. Deep-convert to array
+ // beforehand.
$entity->data['json'][$row->language] = json_decode(json_encode($row), TRUE);
- // Add i18n titles, since the Title field of Scald is not translatable we store this inside the data variable
+ // Add i18n titles, since the Title field of Scald is not translatable we
+ // store this inside the data variable.
$entity->data['title'][$row->language] = $row->title;
- // Add the key as well
+ // Add the key as well.
$entity->data['key'] = $row->{$this->key};
- // Add a timestamp of the last update
+ // Add a timestamp of the last update.
$entity->data['last_update'] = time();
- // Add the name of the module that requested the import
+ // Add the name of the module that requested the import.
if (!empty($this->module)) {
$entity->data['module'] = $this->module;
}
@@ -210,10 +224,10 @@ class ScaldOPAMigration extends Migration {
* Called immediately after the complete destination object is saved.
*/
public function complete($entity, $row) {
- // Find out if this was a new atom or the updated of an existing one
+ // Find out if this was a new atom or the updated of an existing one.
$hook = isset($entity->original) ? 'update' : 'insert';
- // Logging
+ // Logging.
$msg = 'Video %verb: %no_emiarte %kind, %title.';
$vars = array(
'%verb' => isset($entity->original) ? 'updated' : 'created',
@@ -221,10 +235,10 @@ class ScaldOPAMigration extends Migration {
'%kind' => $row->kind,
'%title' => $entity->title,
);
- $link = l('Atom ' . $entity->sid . ' debug', 'atom/' . $entity->sid . '/debug');
+ $link = l(t('Atom @entity_sid debug', 'atom/@entity_sid/debug'), array('@entity_sid' => $entity->sid));
watchdog('scald_opa_migrate', $msg, $vars, WATCHDOG_DEBUG, $link);
- // Store processed atoms
+ // Store processed atoms.
$this->processedAtoms[] = $entity;
// Notify that we have a new or updated scald atom.
@@ -232,10 +246,10 @@ class ScaldOPAMigration extends Migration {
}
/**
- * Called when the import has finished
+ * Called when the import has finished.
*/
public function postImport() {
- // Notify that the import has finished
+ // Notify that the import has finished.
module_invoke_all('scald_opa_migrate_complete', $this->processedAtoms);
}
@@ -259,7 +273,7 @@ class ScaldOPAMigration extends Migration {
$key = array();
foreach ($source_key as $field_name => $field_schema) {
- // Start our unique key with the emission number
+ // Start our unique key with the emission number.
$val = $row->emNumber;
if (isset($row->reassemblyRef) && $row->reassemblyRef) {
@@ -333,7 +347,8 @@ class ScaldOPAMigration extends Migration {
protected function parseThumbnail($row) {
if (isset($row->mainImage)) {
$url = $row->mainImage['url'];
- // Import the external image and save the fid as a file with the emmission number
+ // Import the external image and save the fid as a file with the emmission
+ // number.
$fid = $this->importExternalImage($row->mainImage['url'], $row->mainImage['name']);
$row->thumbnail_fid = $fid;
}
@@ -352,7 +367,9 @@ class ScaldOPAMigration extends Migration {
}
/**
- * Parse thesaurus terms. We use the hook_scald_opa_migrate_mapping_info
+ * Parse thesaurus terms.
+ *
+ * We use the hook_scald_opa_migrate_mapping_info
* to allow other modules to provide taxonomy mapping settings.
*
* @param [type] $row [description]
@@ -367,24 +384,31 @@ class ScaldOPAMigration extends Migration {
if (!empty($row->thesaurus)) {
$terms_import = array();
- // Loop through all thesaurus terms and save the terms we should import
+ // Loop through all thesaurus terms and save the terms we should
+ // import.
foreach ($row->thesaurus as $term) {
// Provide support for previous implementations of this hook.
- // Can be removed once everyone defines "parents" instead of "parent"
+ // Can be removed once everyone defines "parents" instead of
+ // "parent".
if (!isset($mapping['parents']) && isset($mapping['parent'])) {
$mapping['parents'] = array($mapping['parent'] => array('inclusive' => FALSE));
}
- // See if this term qualifies for import. It needs to have at least one of the mapping parents as its parents
+ // See if this term qualifies for import. It needs to have at least
+ // one of the mapping parents as its parents.
$common_parents = array_intersect(array_keys($mapping['parents']), $term['parents']);
if (!empty($common_parents)) {
// We do have common parents so this term should be imported!
- // We take the first common parent in case there are multiple, the other is disregarded.
+ // We take the first common parent in case there are multiple, the
+ // other is disregarded.
$common_parent = reset($common_parents);
- // Check if we need to be inclusive or exclusive with the parent, i.e. import the parent as well. By default we are *not* inclusive, i.e. we use the parent merely as the top level, saying that we'll import all the children
+ // Check if we need to be inclusive or exclusive with the parent,
+ // i.e. import the parent as well. By default we are *not*
+ // inclusive, i.e. we use the parent merely as the top level,
+ // saying that we'll import all the children.
$inclusive = isset($mapping['parents'][$common_parent]['inclusive']) ? $mapping['parents'][$common_parent]['inclusive'] : FALSE;
if ($term['code'] == $common_parent && !$inclusive) {
@@ -393,19 +417,21 @@ class ScaldOPAMigration extends Migration {
}
// We now have all the terms that should be imported
- // We are going to slice the parents to keep only those that are relevant for our import, i.e. the parents until the main parent that is defined in the mapping
+ // We are going to slice the parents to keep only those that are
+ // relevant for our import, i.e. the parents until the main parent
+ // that is defined in the mapping.
$parents_sliced = array();
foreach ($term['parents'] as $i => $parent) {
if ($parent == $common_parent) {
if ($inclusive) {
- // If this is inclusive, we need to take the parent itself
+ // If this is inclusive, we need to take the parent itself.
$parents_sliced = array_slice($term['parents'], $i);
}
else {
- // Skip the parents itself, since this is not inclusive
- $parents_sliced = array_slice($term['parents'], $i+1);
+ // Skip the parents itself, since this is not inclusive.
+ $parents_sliced = array_slice($term['parents'], $i + 1);
}
- // Take off the last parents, it is the current term id
+ // Take off the last parents, it is the current term id.
array_pop($parents_sliced);
}
}
@@ -416,15 +442,15 @@ class ScaldOPAMigration extends Migration {
}
// We now have all the terms that we need to import.
- // Sort the terms to make sure we import first parents, then children
+ // Sort the terms to make sure we import first parents, then children.
usort($terms_import, array('ScaldOPAMigration', 'sort_terms'));
- // Sync up terms, and take the returned tids
- // We also make sure to include the parents
+ // Sync up terms, and take the returned tids.
+ // We also make sure to include the parents.
$key = $mapping['field_name'] . '_tids';
$row->$key = $this->fetchTerms($mapping['vocabulary_name'], $terms_import, $row->language);
- // Add the taxonomy mappings
+ // Add the taxonomy mappings.
if (!empty($row->$key)) {
$this->addFieldMapping($mapping['field_name'], $key, FALSE);
$this->addFieldMapping($mapping['field_name'] . ':source_type', NULL, FALSE)
@@ -439,7 +465,9 @@ class ScaldOPAMigration extends Migration {
}
/**
- * Parse categories terms. We use the hook_scald_opa_migrate_mapping_info
+ * Parse categories terms.
+ *
+ * We use the hook_scald_opa_migrate_mapping_info
* to allow other modules to provide opa_categories mapping settings.
*
* @param [type] $row [description]
@@ -452,21 +480,21 @@ class ScaldOPAMigration extends Migration {
// Each mapping provides a taxonomy - thesaurus mapping.
foreach ($mappings as $mapping) {
if (!empty($row->categories)) {
- // Sync up terms, and take the returned tids
- // We also make sure to include the parents
+ // Sync up terms, and take the returned tids.
+ // We also make sure to include the parents.
$key = $mapping['field_name'] . '_tids';
if (function_exists('categories_opa_migrate_fetch_terms')) {
$row->$key = categories_opa_migrate_fetch_terms($mapping['vocabulary_name'], $row->categories);
- // Add the taxonomy mappings
+ // Add the taxonomy mappings.
if (!empty($row->$key)) {
$this->addFieldMapping($mapping['field_name'], $key, FALSE);
$this->addFieldMapping($mapping['field_name'] . ':source_type', NULL, FALSE)
->defaultValue(TRUE);
}
else {
- // Flag this atom for update when the categories opa run
+ // Flag this atom for update when the categories opa run.
$this->addFieldMapping($mapping['update_field'], NULL, TRUE)->defaultValue(1);
}
}
@@ -499,14 +527,15 @@ class ScaldOPAMigration extends Migration {
foreach ($terms as $term) {
$tids = array_merge($tids, $this->fetchTaxonomyTerm($vocabulary_name, $term, $language));
}
- // Clean up any duplicates, we might have duplicate parents for example
+ // Clean up any duplicates, we might have duplicate parents for example.
$tids = array_unique($tids);
return $tids;
}
/**
- * Checks whether a term identified by name and vocabulary exists. Creates a
- * new term if it does not exist.
+ * Checks whether a term identified by name and vocabulary exists.
+ *
+ * Creates a new term if it does not exist.
*
* @param [type] $vocabulary_name Machine-name
* @param [type] $data Contains name, apios_id and any other fields
@@ -521,12 +550,13 @@ class ScaldOPAMigration extends Migration {
if (!$term) {
$term = new stdClass();
$term->vid = $vocabulary->vid;
- $term->uid = 1; // assign to admin
+ $term->uid = 1; // Assign to admin.
$term->name = $data['label'];
$term->field_apios_id[LANGUAGE_NONE][0]['value'] = $data['code'];
}
- // Disable OPA parent relationships checking until we can trust parent relationships again and won't get into a loop.
+ // Disable OPA parent relationships checking until we can trust parent
+ // relationships again and won't get into a loop.
// @see http://jira.newtech.arte.tv/browse/AFOUND-320
// Add a parent relationship if any is given
// if (!empty($data['parents_sliced'])) {
@@ -538,7 +568,7 @@ class ScaldOPAMigration extends Migration {
// }
// }
- // Update the term
+ // Update the term.
$this->updateTaxonomyTerm($term, $data, $language);
$tids[] = $term->tid;
@@ -559,12 +589,12 @@ class ScaldOPAMigration extends Migration {
$term->name = $data['label'];
}
- // Save the taxonomy term
+ // Save the taxonomy term.
taxonomy_term_save($term);
- // Add i18n support for the given languages
- // This assumes that we have enabled the module i18n_taxonomy and that we use
- // the "Localize" i18n method for this taxonomy.
+ // Add i18n support for the given languages.
+ // This assumes that we have enabled the module i18n_taxonomy and that we
+ // use the "Localize" i18n method for this taxonomy.
// @todo add these checks in the administration pages
if (module_exists('i18n_taxonomy') && $language != $this->defaultLanguage) {
i18n_string_translation_update('taxonomy:term:' . $term->tid . ':name', $data['label'], $language);
@@ -580,34 +610,36 @@ class ScaldOPAMigration extends Migration {
protected function validateDate($date, $field) {
$timestamp = strtotime($date);
if ($timestamp >= 2147483647) {
- // We've hit the infamous Y2038 problem. http://en.wikipedia.org/wiki/Year_2038_problem
- // Drupal unfortunately isn't yet built for post 2038 dates, so let's return a far away date in the future
- $this->queueMessage(t('Hit the Y2038 bug for the date %date in field %field. Returned a far away date instead.', array('%date' => $date , '%field' => $field)), MigrationBase::MESSAGE_WARNING);
+ // We've hit the infamous Y2038 problem.
+ // @see http://en.wikipedia.org/wiki/Year_2038_problem
+ // Drupal unfortunately isn't yet built for post 2038 dates,
+ // so let's return a far away date in the future.
+ $this->queueMessage(t('Hit the Y2038 bug for the date %date in field %field. Returned a far away date instead.', array('%date' => $date, '%field' => $field)), MigrationBase::MESSAGE_WARNING);
return '2037-10-14T23:59:00';
}
- // Validation passed, return the original date
+ // Validation passed, return the original date.
return $date;
}
/**
- * Imports an external image into a local Drupal image files (JPG)
+ * Imports an external image into a local Drupal image files (JPG).
*
* @param [type] $url The remote URL
* @param [type] $base_id The base id that will be used to identify this file
* @return [type] fid from imported file
*/
protected function importExternalImage($url, $filename) {
- // Directory in which to save the images
+ // Directory in which to save the images.
$file_directory = 'public://atoms/image/opa';
- // Fetch the remote image from the URL and save as a local Drupal JPG image
+ // Fetch the remote image from the URL and save as a local Drupal JPG image.
$image = @file_get_contents($url);
if ($image) {
$file = file_save_data($image, $file_directory . '/' . $filename, FILE_EXISTS_REPLACE);
- // We return the fid which will be used in the mapping
+ // We return the fid which will be used in the mapping.
if ($file) {
return $file->fid;
}
@@ -623,12 +655,17 @@ class ScaldOPAMigration extends Migration {
}
/**
- * Auxiliary function to look up a taxonomy term by Apios ID, assumes SQL
- * storage backend.
+ * Auxiliary function to look up a taxonomy term by Apios ID.
+ *
+ * Assumes SQL storage backend.
+ *
+ * @param Object $vocabulary
+ * The vocabulary.
+ * @param Integer $apios_id
+ * The Apios ID of the term to look up.
*
- * @param Object $vocabulary the vocabulary.
- * @param Integer $apios_id the Apios ID of the term to look up.
- * @return Mixed the term ID or FALSE if term was not found.
+ * @return Mixed
+ * The term ID or FALSE if term was not found.
*/
protected function getTaxonomyTermByApiosID($vocabulary, $apios_id) {
return scald_opa_migrate_get_term_by_apios_id($vocabulary->machine_name, $apios_id);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment