TYPO3 Surf deployment for TYPO3 CMS with one target node and getting the database and files from another server

AdditionalConfiguration.php

<?php

$GLOBALS['TYPO3_CONF_VARS']['BE']['installToolPassword'] = '$P$CEF2NP3kCoIHwDupr8RiYUMQ7/Rp.j.';

$GLOBALS['TYPO3_CONF_VARS']['DB']['database'] = 'projects_my_onenodeintegration';
$GLOBALS['TYPO3_CONF_VARS']['DB']['username'] = 'user';
$GLOBALS['TYPO3_CONF_VARS']['DB']['password'] = 'secret';

$GLOBALS['TYPO3_CONF_VARS']['SYS']['sitename'] = 'My OneNode Integration';

$GLOBALS['TYPO3_CONF_VARS']['BE']['debug'] = TRUE;

$GLOBALS['TYPO3_CONF_VARS']['FE']['debug'] = TRUE;

$GLOBALS['TYPO3_CONF_VARS']['SYS']['devIPmask'] = '*';
$GLOBALS['TYPO3_CONF_VARS']['SYS']['displayErrors'] = TRUE;
$GLOBALS['TYPO3_CONF_VARS']['SYS']['enableDeprecationLog'] = 'file';
$GLOBALS['TYPO3_CONF_VARS']['SYS']['sqlDebug'] = 1;
$GLOBALS['TYPO3_CONF_VARS']['SYS']['systemLogLevel'] = 0;

$localconfDir = dirname(__FILE__);
if (file_exists($localconfDir . '/iniset.php')) {
	include_once($localconfDir . '/iniset.php');
}
?>

config.php

<?php
$deploymentName = 'My.OneNode.IntegrationDeployment';
$applicationName = 'My.OneNode';
// SSH username. Authentication via private/public key
$username = 'user';
// Url to the project git repository
$repositoryUrl = 'ssh://host.com/git/my/onenode.git';
// Path to the TYPO3 source on the server
$typo3srcPath = '/data/TYPO3/CMS/TYPO3_6-2-3';
// Path to the shared/editorial files on the server
$filesPath = '/data/projects/My/OneNodeIntegration/files';
// URLs for smoke tests
$backendURL = 'http://integration.onenode.my.projects.localhost/typo3';
$backendURLAdditionalCurlParameters = '';
$frontendURL = 'http://integration.onenode.my.projects.localhost';
$frontendURLAdditionalCurlParameters = '';
// The deployment path is not the document root!
// The document root has to be: '[deploymentPath]/releases/current'.
$deploymentPath = '/data/projects/My/OneNodeIntegration/deployment';

/**
 * The host on which we want to do the deployment.
 */
// A speaking name to identify this host
$host = 'integration.onenode';
// Hostname, used to connect via ssh
$hostname = 'integration.onenode.my.projects.localhost';
// Port, used to connect via ssh
$port = '22';
// Database connection configuration on this host (f.e. used for backups)
$dbconHost = 'localhost';
$dbconPort = '3306';
$dbconUsername = 'user';
$dbconPassword = 'secret';
$dbconDatabase = 'projects_my_onenodeintegration';

/**
 * SSH and database connection configuration to the live server to get the current live database
 */
$liveSshUsername = 'user';
$liveSshHost = 'onenode';
$liveSshPort = '22';
$liveDbconHost = 'localhost';
$liveDbconPort = '3306';
$liveDbconUsername = 'user';
$liveDbconPassword = 'secret'; //I don't like passwords in config files ;-). The shell will ask you for it.
$liveDbconDatabase = 'projects_my_onenode';
$liveFilesPath = '/data/projects/My/OneNode/files';
?>

iniset.php

<?php

ini_set('realpath_cache_size', '0');
ini_set('realpath_cache_ttl', '0');
clearstatcache();
clearstatcache(TRUE);

?>

LocalPageTSconfig.ts

TCEMAIN.previewDomain = integration.onenode.my.projects.localhost

My.OneNode.IntegrationDeployment.php

<?php

/** @var \TYPO3\Surf\Domain\Model\Deployment $deployment */

/**
 * config.php is not versionized with git so we can have different configs to develop this deployment script
 */
$deploymentConfigPathAndFilename = FLOW_PATH_ROOT . 'Build/Surf/Resources/My.OneNode.IntegrationDeployment/config.php';
if (!file_exists($deploymentConfigPathAndFilename)) {
	exit(sprintf("The deployment config file %s does not exist.\n", $deploymentConfigPathAndFilename));
}
require_once($deploymentConfigPathAndFilename);

/**
 * Node
 */
$node = new \TYPO3\Surf\Domain\Model\Node($host);
$node->setHostname($hostname);
$node->setOption('username', $username);
$node->setOption('port', $port);

/**
 * Application
 */
$application = new \TYPO3\Surf\Application\BaseApplication($applicationName);
$application->setOption('repositoryUrl', $repositoryUrl);
$application->setOption('transferMethod', 'rsync');
$application->setOption('rsyncFlags', '--recursive --times --perms --links --delete --delete-excluded --exclude \'.git\' --exclude \'.sass-cache\'');
$application->setOption('packageMethod', 'git');
$application->setOption('updateMethod', NULL);
$application->setOption('branch', 'integration');
$application->setDeploymentPath($deploymentPath);
$application->addNode($node);
$deployment->addApplication($application);

/**
 * Create workflow
 */
$workflow = new \BGM\Deployment\Domain\Model\TYPO3CMSWorkflow();
$deployment->setWorkflow($workflow);

/**
 * Compile CSS
 */
$compilecssOptions = array(
	'command' => '
		cd "{workspacePath}" &&
		BUNDLE_GEMFILE=typo3conf/ext/bgm_theme_project/Resources/Private/Gemfile bundle install &&
		BUNDLE_GEMFILE=typo3conf/ext/bgm_theme_project/Resources/Private/Gemfile bundle exec compass compile typo3conf/ext/bgm_theme_project/Resources/Private/ --force',
);
$workflow->defineTask('my.onenode.integrationdeployment:compilecss', 'typo3.surf:localshell', $compilecssOptions);
$workflow->afterStage('package', 'my.onenode.integrationdeployment:compilecss');

/**
 * Set release identifier file
 */
$setreleaseidentifierfileOptions = array(
	'command' => '
		echo ' . escapeshellarg('<?php echo \'' . $deployment->getReleaseIdentifier() . '\'; ?>') . ' > {releasePath}/Release_' . $deployment->getReleaseIdentifier() . '.php',
);
$workflow->defineTask('my.onenode.integrationdeployment:setreleaseidentifierfile', 'typo3.surf:shell', $setreleaseidentifierfileOptions);
$workflow->afterStage('transfer', 'my.onenode.integrationdeployment:setreleaseidentifierfile');

/**
 * Set AdditionalConfiguration.php
 *
 * AdditionalConfiguration.php has to exist and has to contain at least "<?php if (!defined ('TYPO3_MODE')) die('Access denied.'); ?>"
 */
$setadditionalconfigurationOptions = array(
	'command' => '
		echo ' . escapeshellarg(\TYPO3\Flow\Utility\Files::getFileContents(FLOW_PATH_ROOT . 'Build/Surf/Resources/My.OneNode.IntegrationDeployment/AdditionalConfiguration.php')) . ' > {releasePath}/typo3conf/AdditionalConfiguration.php',
);
$workflow->defineTask('my.onenode.integrationdeployment:setadditionalconfiguration', 'typo3.surf:shell', $setadditionalconfigurationOptions);
$workflow->afterStage('transfer', 'my.onenode.integrationdeployment:setadditionalconfiguration');

/**
 * Set iniset.php
 *
 * iniset.php has to exist and has to contain at least "<?php  ?>"
 */
$setinisetOptions = array(
	'command' => '
		echo ' . escapeshellarg(\TYPO3\Flow\Utility\Files::getFileContents(FLOW_PATH_ROOT . 'Build/Surf/Resources/My.OneNode.IntegrationDeployment/iniset.php')) . ' > {releasePath}/typo3conf/iniset.php;
		echo ' . escapeshellarg(\TYPO3\Flow\Utility\Files::getFileContents(FLOW_PATH_ROOT . 'Build/Surf/Resources/My.OneNode.IntegrationDeployment/iniset.php')) . ' > {currentPath}/typo3conf/iniset.php',
	'rollbackCommand' => '
		rm -f {currentPath}/typo3conf/iniset.php',
);
$workflow->defineTask('my.onenode.integrationdeployment:setiniset', 'typo3.surf:shell', $setinisetOptions);
$workflow->afterStage('transfer', 'my.onenode.integrationdeployment:setiniset');

/**
 * Set symlinks to typo3_src and shared files
 */
$setsymlinksOptions = array(
	'command' => '
		ln -s ' . $typo3srcPath . ' {releasePath}/typo3_src &&
		ln -s ' . $filesPath . ' {releasePath}/files',
);
$workflow->defineTask('my.onenode.integrationdeployment:setsymlinks', 'typo3.surf:shell', $setsymlinksOptions);
$workflow->afterStage('transfer', 'my.onenode.integrationdeployment:setsymlinks');

/**
 * Set backend lock on next live
 */
$setbackendlockOptions = array(
	'command' => '
		{releasePath}/typo3/cli_dispatch.phpsh lowlevel_admin setBElock;
		{currentPath}/typo3/cli_dispatch.phpsh lowlevel_admin setBElock;
		true',
	'rollbackCommand' => '
		{releasePath}/typo3/cli_dispatch.phpsh lowlevel_admin clearBElock;
		{currentPath}/typo3/cli_dispatch.phpsh lowlevel_admin clearBElock;
		true',
);
$workflow->defineTask('my.onenode.integrationdeployment:setbackendlock', 'typo3.surf:shell', $setbackendlockOptions);
$workflow->beforeStage('migrate', 'my.onenode.integrationdeployment:setbackendlock');

/**
 * Do database backup
 */
$dodatabasebackupOptions = array(
	'command' => '
		mysqldump -u' . $dbconUsername . ' -p' . $dbconPassword . ' -h' . $dbconHost . ' -P' . $dbconPort . ' --default-character-set=utf8 --opt --skip-lock-tables --skip-add-locks --lock-tables=false --single-transaction --quick ' . $dbconDatabase . ' > {currentPath}/deploymentbackup.sql',
	'rollbackCommand' => '
		mysql -u' . $dbconUsername . ' -p' . $dbconPassword . ' -h' . $dbconHost . ' -P' . $dbconPort . ' ' . $dbconDatabase . ' < {currentPath}/deploymentbackup.sql &&
		rm -f {currentPath}/deploymentbackup.sql;
		true',
);
$workflow->defineTask('my.onenode.integrationdeployment:dodatabasebackup', 'typo3.surf:shell', $dodatabasebackupOptions);
$workflow->beforeStage('migrate', 'my.onenode.integrationdeployment:dodatabasebackup');

/**
 * Get live database from live server
 */
echo "\n";
echo "###########################################################\n";
echo "# Get live user data (database & files) from live server? #\n";
echo "###########################################################\n";
echo "This can take some time!\n";
echo "Type 'yes' or 'no' and hit ENTER:\n";
echo "\n";
$handle = fopen("php://stdin", "r");
$line = fgets($handle);
$getLiveDatabase = FALSE;
if (trim($line) == 'yes') {
	$getLiveDatabase = TRUE;
	echo "Getting data is activated.\n";
	echo "\n";
	$getlivedatabaseOptions = array(
		'command' => '
			mysqldump -u' . $liveDbconUsername . ' -p' . $liveDbconPassword . ' -h' . $liveDbconHost . ' -P' . $liveDbconPort . ' --default-character-set=utf8 --opt --skip-lock-tables --skip-add-locks --lock-tables=false --single-transaction --quick ' . $liveDbconDatabase . ' | gzip -3 -c > {releasePath}/livedatabase.sql.gz &&
			gunzip {releasePath}/livedatabase.sql.gz &&
			mysql -u' . $dbconUsername . ' -p' . $dbconPassword . ' -h' . $dbconHost . ' -P' . $dbconPort . ' ' . $dbconDatabase . ' < {releasePath}/livedatabase.sql &&
			rm -f {releasePath}/livedatabase.sql',
	);
	$workflow->defineTask('my.onenode.integrationdeployment:getlivedatabase', 'typo3.surf:shell', $getlivedatabaseOptions);
	$workflow->beforeStage('migrate', 'my.onenode.integrationdeployment:getlivedatabase');

	$setdomainsOptions = array(
		'command' => '
			echo "
				INSERT sys_domain (pid, tstamp, crdate, domainName, forced, sorting)
				VALUES
					(1, UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), \'integration.onenode.my.projects.localhost\', 0, (SELECT MAX(sd.sorting)+1 FROM sys_domain sd));
			" | mysql -u' . $dbconUsername . ' -p' . $dbconPassword . ' -h' . $dbconHost . ' -P' . $dbconPort . ' ' . $dbconDatabase,
	);
	$workflow->defineTask('my.onenode.integrationdeployment:setdomains', 'typo3.surf:shell', $setdomainsOptions);
	$workflow->beforeStage('migrate', 'my.onenode.integrationdeployment:setdomains');

	$getlivefilesOptions = array(
		'command' => '
			rsync --progress -avz ' . $liveFilesPath . '/ ' . $filesPath . '',
	);
	$workflow->defineTask('my.onenode.integrationdeployment:getlivefiles', 'typo3.surf:shell', $getlivefilesOptions);
	$workflow->beforeStage('migrate', 'my.onenode.integrationdeployment:getlivefiles');
} else {
	echo "Getting data is not activated.\n";
	echo "Be careful, perhaps you are testing with a really old database and old files!\n";
	echo "I hope you know, what you are doing...\n";
	echo "\n";
}

/**
 * Do database compare on next live
 *
 * 1=ACTION_UPDATE_CLEAR_TABLE ,2=ACTION_UPDATE_ADD ,3=ACTION_UPDATE_CHANGE ,4=ACTION_UPDATE_CREATE_TABLE ,5=ACTION_REMOVE_CHANGE ,6=ACTION_REMOVE_DROP ,7=ACTION_REMOVE_CHANGE_TABLE ,8=ACTION_REMOVE_DROP_TABLE
 *
 * @see EXT:coreapi/Classes/Service/DatabaseApiService.php
 */
$dodatabasecompareOptions = array(
	'command' => '
		sync;
		{releasePath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearallactiveopcodecache;
		{releasePath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearconfigurationcache;
		{releasePath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearsystemcache;
		{releasePath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearpagecache;
		{releasePath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearallcaches;
		{releasePath}/typo3/cli_dispatch.phpsh extbase databaseapi:databasecompare 2,3,4',
	'rollbackCommand' => '
		sync;
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearallactiveopcodecache;
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearconfigurationcache;
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearsystemcache;
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearpagecache;
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearallcaches;
		{currentPath}/typo3/cli_dispatch.phpsh extbase databaseapi:databasecompare 2,3,4',
);
$workflow->defineTask('my.onenode.integrationdeployment:dodatabasecompare', 'typo3.surf:shell', $dodatabasecompareOptions);
$workflow->addTask('my.onenode.integrationdeployment:dodatabasecompare', 'migrate');

/**
 * Symlink shared typo3temp in releasePath
 *
 * On rollback, reload apache2
 */
$copytempassetsOptions = array(
	'command' => '
		rm -rf {releasePath}/typo3temp;
		ln -s {sharedPath}/typo3temp {releasePath}/typo3temp;
		sync;
		true',
	'rollbackCommand' => '
		sync;
		sudo /etc/init.d/httpd reload;
		rm -rf {currentPath}/typo3temp/Cache;
		rm -rf {currentPath}/typo3temp/locks;
		true',
);
$workflow->defineTask('my.onenode.integrationdeployment:copytempassets', 'typo3.surf:shell', $copytempassetsOptions);
$workflow->addTask('my.onenode.integrationdeployment:copytempassets', 'finalize');

/**
 * Fix file permissions
 */
$fixfilepermissionOptions = array(
	'command' => '
		chown -R apacheuser:sharedgroup {releasePath};
		find {releasePath}/. -type f -print0 | xargs -0 chmod 664;
		find {releasePath}/. -type d -print0 | xargs -0 chmod 775',
);
$workflow->defineTask('my.onenode.integrationdeployment:fixfilepermission', 'typo3.surf:shell', $fixfilepermissionOptions);
$workflow->addTask('my.onenode.integrationdeployment:fixfilepermission', 'finalize');

/**
 * Reload Apache
 */
$reloadapacheOptions = array(
	'command' => '
		sync;
		/etc/init.d/httpd reload;
		rm -rf {currentPath}/typo3temp/Cache;
		rm -rf {currentPath}/typo3temp/locks;
		true',
);
$workflow->defineTask('my.onenode.integrationdeployment:reloadapache', 'typo3.surf:shell', $reloadapacheOptions);
$workflow->afterStage('switch', 'my.onenode.integrationdeployment:reloadapache');

/**
 * Wait for symlink switch recognized by all caches
 */
$waitforsymlinkswitchOptions = array(
	'command' => '
		testcurl=$(curl -Is \'http://integration.onenode.my.projects.localhost/Release_' . $deployment->getReleaseIdentifier() . '.php\' | head -n 1 | grep \'200\' | wc -l);
		echo "testcurl $testcurl";
		counter=0;
		while [ $testcurl -ne 1 ];
			do echo \'waitforsymlinkswitch\';
			sleep 1;
			testcurl=$(curl -Is \'http://integration.onenode.my.projects.localhost/Release_' . $deployment->getReleaseIdentifier() . '.php\' | head -n 1 | grep \'200\' | wc -l);
			echo "testcurl $testcurl";
			counter=$(( counter+1 ));
			echo "counter $counter";
		done;',
);
$workflow->defineTask('my.onenode.integrationdeployment:waitforsymlinkswitch', 'typo3.surf:shell', $waitforsymlinkswitchOptions);
$workflow->addTask('my.onenode.integrationdeployment:waitforsymlinkswitch', 'warmup');

/**
 * Clear all caches
 */
$clearallcachesafterswitchOptions = array(
	'command' => '
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearallactiveopcodecache;
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearconfigurationcache;
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearsystemcache;
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearpagecache;
		{currentPath}/typo3/cli_dispatch.phpsh extbase cacheapi:clearallcaches;
		true;',
);
$workflow->defineTask('my.onenode.integrationdeployment:clearallcachesafterswitch', 'typo3.surf:shell', $clearallcachesafterswitchOptions);
$workflow->addTask('my.onenode.integrationdeployment:clearallcachesafterswitch', 'warmup');

/**
 * Remove backend locks
 */
$removebackendlockOptions = array(
	'command' => '
		{currentPath}/typo3/cli_dispatch.phpsh lowlevel_admin clearBElock;
		{previousPath}/typo3/cli_dispatch.phpsh lowlevel_admin clearBElock;
		true;',
);
$workflow->defineTask('my.onenode.integrationdeployment:removebackendlock', 'typo3.surf:shell', $removebackendlockOptions);
$workflow->beforeStage('cleanup', 'my.onenode.integrationdeployment:removebackendlock');

/**
 * Fix file permissions
 */
$fixfilepermission2Options = array(
	'command' => '
		chown -R apacheuser:sharedgroup {releasePath};
		chown -R apacheuser:sharedgroup {releasePath}/typo3temp/;
		find {releasePath}/. -type f -print0 | xargs -0 chmod 664;
		find {releasePath}/typo3temp/. -type f -print0 | xargs -0 chmod 664;
		find {releasePath}/. -type d -print0 | xargs -0 chmod 775;
		find {releasePath}/typo3temp/. -type d -print0 | xargs -0 chmod 775',
);
$workflow->defineTask('my.onenode.integrationdeployment:fixfilepermission2', 'typo3.surf:shell', $fixfilepermission2Options);
$workflow->beforeStage('cleanup', 'my.onenode.integrationdeployment:fixfilepermission2');

/**
 * Set git tag
 */
$settagOptions = array(
	'tagName' => '{deploymentName}-{releaseIdentifier}',
	'description' => 'Deployed with {deploymentName} at ' . date('Y-m-d H:i') . '
	Release identifier is {releaseIdentifier}'
		. ($getLiveDatabase
			? '

	Getting live data was activated.'
			: ''
		),
	'pushTag' => TRUE,
);
$workflow->defineTask('my.onenode.integrationdeployment:settag', 'bgm.deployment:git:tag', $settagOptions);
$workflow->beforeStage('cleanup', 'my.onenode.integrationdeployment:settag');

/**
 * Remove iniset.php
 */
$removeinisetOptions = array(
	'command' => '
		rm -f {currentPath}/typo3conf/iniset.php;
		rm -f {previousPath}/typo3conf/iniset.php;
		true',
);
$workflow->defineTask('my.onenode.integrationdeployment:removeiniset', 'typo3.surf:shell', $removeinisetOptions);
$workflow->addTask('my.onenode.integrationdeployment:removeiniset', 'cleanup');

/**
 * Compress database backup
 */
$compressdatabasebackupOptions = array(
	'command' => '
		gzip {previousPath}/deploymentbackup.sql;
		true',
);
$workflow->defineTask('my.onenode.integrationdeployment:compressdatabasebackup', 'typo3.surf:shell', $compressdatabasebackupOptions);
$workflow->addTask('my.onenode.integrationdeployment:compressdatabasebackup', 'cleanup');

/**
 * Delete all releases older than 6 days, but keeps at least 5 releases
 */
$deleteoldreleasesOptions = array(
	'command' => '
		ls -At1 --ignore=previous --ignore=current --ignore=next {deploymentPath}/releases/ | tail -n +5 | xargs -n1 basename|xargs -n1 -I{} find {deploymentPath}/releases/ -name {} -type d -mtime +6 | xargs -r rm -rf;
		true',
);
$workflow->defineTask('my.onenode.integrationdeployment:deleteoldreleases', 'typo3.surf:shell', $deleteoldreleasesOptions);
$workflow->addTask('my.onenode.integrationdeployment:deleteoldreleases', 'cleanup');

?>