TYPO3 EXT:flux field.file -> field.inline.fal migration script

FluxFalUpdateWizard.php

<?php
namespace MyVendor\Sitepackage\Updates;

/**
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */
use Doctrine\DBAL\DBALException;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use TYPO3\CMS\Core\Core\Environment;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
use TYPO3\CMS\Core\Registry;
use TYPO3\CMS\Core\Resource\File;
use TYPO3\CMS\Core\Resource\ResourceStorage;
use TYPO3\CMS\Core\Resource\StorageRepository;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Install\Updates\UpgradeWizardInterface;

/**
 * Upgrade wizard which goes through all files referenced in fe_users::image
 * and creates sys_file records as well as sys_file_reference records for each hit.
 * @internal This class is only meant to be used within EXT:install and is not part of the TYPO3 Core API.
 */
class FluxFalUpdateWizard implements UpgradeWizardInterface, LoggerAwareInterface
{
    use LoggerAwareTrait;

    /**
     * Number of records fetched per database query
     * Used to prevent memory overflows for huge databases
     */
    const RECORDS_PER_QUERY = 5000;

    /**
     * @var ResourceStorage
     */
    protected $storage;

    /**
     * Table to migrate records from
     *
     * @var string
     */
    protected $table = 'tt_content';

    /**
     * Table field holding the migration to be
     *
     * @var string
     */
    protected $fieldToMigrate = 'pi_flexform';

    /**
     * all fields to migrate in pi_flexform
     */
    protected $piFlexformFields = [];

    /**
     * namespace for FLUX CEs
     */
    protected $nameSpaceFluxElements = '';

    /**
     * the source file resides here
     *
     * @var string
     */
    protected $sourcePath = '/';

    /**
     * target folder after migration
     * Relative to fileadmin
     *
     * @var string
     */
    protected $targetPath = '/';

    /**
     * @var Registry
     */
    protected $registry;

    /**
     * @var string
     */
    protected $registryNamespace = 'FluxFalUpdateWizard';

    /**
     * @var array
     */
    protected $recordOffset = [];

    /**
     * @return string Unique identifier of this updater
     */
    public function getIdentifier(): string
    {
        return 'fluxFalUpdateWizard';
    }

    /**
     * @return string Title of this updater
     */
    public function getTitle(): string
    {
        return 'Migrate file relations from Flux CEs to FAL';
    }

    /**
     * @return string Longer description of this updater
     */
    public function getDescription(): string
    {
        return 'This update wizard goes through all files that are referenced in the fe_users.image'
            . ' field and adds the files to the FAL File Index. It also moves the files from'
            . ' uploads/ to the fileadmin/_migrated/ path.';
    }

    /**
     * Checks if an update is needed
     *
     * @return bool TRUE if an update is needed, FALSE otherwise
     */
    public function updateNecessary(): bool
    {
        $this->registry = GeneralUtility::makeInstance(Registry::class);
        return $this->registry->get($this->registryNamespace, 'recordOffset') === null;
    }

    /**
     * @return string[] All new fields and tables must exist
     */
    public function getPrerequisites(): array
    {
        return [
            DatabaseUpdatedPrerequisite::class
        ];
    }

    /**
     * Performs the database update.
     *
     * @return bool TRUE on success, FALSE on error
     */
    public function executeUpdate(): bool
    {
        try {
            $this->init();
            if (!isset($this->recordOffset[$this->table])) {
                $this->recordOffset[$this->table] = 0;
            }
            do {
                $limit = $this->recordOffset[$this->table] . ',' . self::RECORDS_PER_QUERY;
                $records = $this->getRecordsFromTable($limit);
                foreach ($records as $record) {
                    $this->migrateField($record);
                }
                $this->registry->set($this->registryNamespace, 'recordOffset', $this->recordOffset);
            } while (count($records) === self::RECORDS_PER_QUERY);

            $this->registry->remove($this->registryNamespace, 'recordOffset');

        } catch (\Exception $e) {
            // Silently catch db errors
            echo $e->getMessage();
            return false;
        }
        return true;
    }

    /**
     * Initialize the storage repository.
     */
    protected function init()
    {
        $storages = GeneralUtility::makeInstance(StorageRepository::class)->findAll();
        $this->storage = $storages[0];
        $this->registry = GeneralUtility::makeInstance(Registry::class);
        $this->recordOffset = $this->registry->get($this->registryNamespace, 'recordOffset', []);

        $this->nameSpaceFluxElements = 'MyVendor.Sitepackage';
        $this->piFlexformFields = [
            'MyVendor.Sitepackage:HeadWithArrow.html' => [
                'header-top-file',
                'header-bottom-bg-file',
                'header-image'
            ],
            'MyVendor.Sitepackage:HeaderBoxThreeColumns.html' => [
                'file'
            ],
            'MyVendor.Sitepackage:HeaderBoxFourColumns.html' => [
                'file'
            ],
            'MyVendor.Sitepackage:ImageCompare.html' => [
                'file-left',
                'file-right'
            ],
        ];
    }

    /**
     * Get records from table where the field to migrate is not empty (NOT NULL and != '')
     * and also not numeric (which means that it is migrated)
     *
     * @param int $limit Maximum number records to select
     * @return array
     * @throws \RuntimeException
     */
    protected function getRecordsFromTable($limit)
    {
        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
        /** @var \TYPO3\CMS\Core\Database\Query\QueryBuilder $queryBuilder */
        $queryBuilder = $connectionPool->getQueryBuilderForTable($this->table);
        $queryBuilder->getRestrictions()
            ->removeAll()
            ->add(GeneralUtility::makeInstance(DeletedRestriction::class));
        try {
            $queryBuilder
                ->select('uid', 'pid', 'tx_fed_fcefile', $this->fieldToMigrate)
                ->from($this->table)
                ->where(
                    $queryBuilder->expr()->like('tx_fed_fcefile', $queryBuilder->createNamedParameter($this->nameSpaceFluxElements . '%', \PDO::PARAM_STR))
                )
                ->orWhere(
                    $queryBuilder->expr()->neq('tx_flux_migrated_version', $queryBuilder->createNamedParameter('fal', \PDO::PARAM_STR)),
                    $queryBuilder->expr()->isNull('tx_flux_migrated_version')
                )
                ->orderBy('uid')
                ->setFirstResult($limit);
                return $queryBuilder->execute()->fetchAll();
        } catch (DBALException $e) {
            throw new \RuntimeException(
                'Database query failed. Error was: ' . $e->getPrevious()->getMessage(),
                1476050084
            );
        }
    }

    /**
     * Migrates a single field.
     *
     * @param array $row
     */
    protected function migrateField($row)
    {
        // sanity checks
        if (empty($row[$this->fieldToMigrate])) {
            return;
        }
        if (empty($this->piFlexformFields[$row['tx_fed_fcefile']])) {
            return;
        }

        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);

        $xml = simplexml_load_string(($row[$this->fieldToMigrate]));

        foreach ($this->piFlexformFields[$row['tx_fed_fcefile']] as $piFlexformField)
        {

            $fileNodes = $xml->xpath('//data/sheet/language/field[@index="' . $piFlexformField . '"]/value');
            if (empty($fileNodes)) {
                continue;
            }
            $fileNameWithPath = (string)$fileNodes[0];

            if (substr($fileNameWithPath, 0, 10) !== 'fileadmin/') {
                $this->logger->notice(
                    'File name does not start with /fileadmin - Reference was not migrated.',
                    [
                        'table' => $this->table,
                        'record' => $row,
                        'field' => $this->fieldToMigrate,
                    ]
                );
                continue;
            }
            $fileNameWithPath = substr($fileNameWithPath,10);
            if (!file_exists(Environment::getPublicPath() . '/fileadmin/' . $fileNameWithPath)) {
                $this->logger->notice(
                    'File does not exist - file_exists(). Reference was not migrated.',
                    [
                        'table' => $this->table,
                        'record' => $row,
                        'field' => $this->fieldToMigrate,
                    ]
                );
                continue;
            }
            /** @var File $file */
            $file = $this->storage->getFile($fileNameWithPath);
            if (empty($file)) {
                $this->logger->notice(
                    'File does not exist - getFile(). Reference was not migrated.',
                    [
                        'table' => $this->table,
                        'record' => $row,
                        'field' => $this->fieldToMigrate,
                    ]
                );
                continue;
            }

            $fields = [
                'fieldname' => $piFlexformField,
                'table_local' => 'sys_file',
                'pid' => $row['pid'],
                'uid_foreign' => $row['uid'],
                'uid_local' => $file->getUid(),
                'tablenames' => $this->table,
                'crdate' => time(),
                'tstamp' => time()
            ];
            $queryBuilder = $connectionPool->getQueryBuilderForTable('sys_file_reference');
            $queryBuilder->insert('sys_file_reference')->values($fields)->execute();

            /**
             * this marks the CE as processed
             */
            $queryBuilder = $connectionPool->getQueryBuilderForTable($this->table);
            $queryBuilder->update($this->table)
                         ->set('tx_flux_migrated_version', 'fal')
                         ->where(
                             $queryBuilder->expr()->eq('uid', $queryBuilder->createNamedParameter($row['uid'], \PDO::PARAM_INT))
                         )
                         ->execute();

            //todo: update of the xml and replace of values with simple "1" counter. But data leftover should be of no harm and will be overwritten on next save of record.
        }
    }
}