Hacks to test scap in a vm

mediawiki-vagrant-scap.patch

From 82f70da7e02551e458df7114747802e835a66c5e Mon Sep 17 00:00:00 2001
From: Bryan Davis <bd808@wikimedia.org>
Date: Thu, 13 Feb 2014 13:52:59 -0700
Subject: [PATCH] Hacks to test scap in a vm

A big bag of hacks wrapped in nails and glass, but works for me!

* dsh group files
* lint.php
* mw-deployment-vars.sh
* mwscript
* rsync server configuration
* puppet config
---
 .gitignore                                   |   2 +
 Vagrantfile                                  |  41 +++---
 puppet/files/scap/dsh.mediawiki-installation |   1 +
 puppet/files/scap/dsh.scap-proxies           |   1 +
 puppet/files/scap/lint.php                   |  62 +++++++++
 puppet/files/scap/mw-deployment-vars.sh      |  10 ++
 puppet/files/scap/mwscript                   |  18 +++
 puppet/files/scap/rsync.conf                 |  17 +++
 puppet/files/scap/rsync.default              |   5 +
 puppet/manifests/site.pp                     | 197 +++++++++++++++++++++++++++
 10 files changed, 330 insertions(+), 24 deletions(-)
 create mode 100644 puppet/files/scap/dsh.mediawiki-installation
 create mode 100644 puppet/files/scap/dsh.scap-proxies
 create mode 100755 puppet/files/scap/lint.php
 create mode 100644 puppet/files/scap/mw-deployment-vars.sh
 create mode 100755 puppet/files/scap/mwscript
 create mode 100644 puppet/files/scap/rsync.conf
 create mode 100644 puppet/files/scap/rsync.default

diff --git a/.gitignore b/.gitignore
index 69ed3f7..f1ff501 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,5 @@
 /wikimania_scholarships/
 /vagrant.d/
 /support/packager/output
+/scap
+/mediawiki-config
diff --git a/Vagrantfile b/Vagrantfile
index d96524e..3f7246f 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -99,15 +99,6 @@ Vagrant.configure('2') do |config|
 
     config.vm.network :private_network, ip: settings['static_ip']
 
-    config.vm.network :forwarded_port,
-        guest: 80, host: settings['http_port'].to_i, id: 'http'
-
-    settings['forward_ports'].each { |guest_port,host_port|
-        config.vm.network :forwarded_port,
-            :host => host_port.to_i, :guest => guest_port.to_i,
-            auto_correct: true
-    } unless settings['forward_ports'].nil?
-
     root_share_options = {:id => 'vagrant-root'}
 
     if Vagrant::Util::Platform.windows?
@@ -119,20 +110,22 @@ Vagrant.configure('2') do |config|
         config.nfs.map_gid = Process.gid
     end
 
-    config.vm.synced_folder '.', '/vagrant', root_share_options
-
-    # www-data needs to write to the logs, but doesn't need write
-    # access for all of /vagrant
-    #
-    # TODO (mattflaschen, 2014-05-23): It should also be possible to
-    # use NFS for this (the web server writes to it).  I tried to do
-    # this by putting map_uid and map_gid as 33 (www-data) on the guest,
-    # but it didn't work.  Although the export file looks right the
-    # effective permissions on the guest were still the same as /vagrant.
-    config.vm.synced_folder './logs', '/vagrant/logs',
-        id: 'vagrant-logs',
-        owner: 'www-data',
-        group: 'www-data'
+    config.vm.synced_folder '.', '/vagrant', disabled: true
+
+    config.vm.synced_folder '.', '/vagrant-nfs',
+        id: 'vagrant-root',
+        nfs: true,
+        nfs_export: true,
+        nfs_export_options: ['rw','sync','nohide'],
+        nfs_mount_options: ['noatime'],
+        nfs_udp: true,
+        nfs_version: '3'
+
+    # bindfs needs `vagrant plugin install vagrant-bindfs`
+    config.bindfs.bind_folder '/vagrant-nfs', '/vagrant',
+        owner: 'vagrant',
+        group: 'www-data',
+        perms: 'u=rwx:g=rwx:o=rwx'
 
     config.vm.provider :virtualbox do |vb|
         # See http://www.virtualbox.org/manual/ch08.html for additional options.
@@ -169,7 +162,7 @@ Vagrant.configure('2') do |config|
 
         puppet.facter = $FACTER = {
             'fqdn'               => config.vm.hostname,
-            'forwarded_port'     => settings['http_port'],
+            #'forwarded_port'     => settings['http_port'],
             'shared_apt_cache'   => '/vagrant/apt-cache/',
         }
 
diff --git a/puppet/files/scap/dsh.mediawiki-installation b/puppet/files/scap/dsh.mediawiki-installation
new file mode 100644
index 0000000..2fbb50c
--- /dev/null
+++ b/puppet/files/scap/dsh.mediawiki-installation
@@ -0,0 +1 @@
+localhost
diff --git a/puppet/files/scap/dsh.scap-proxies b/puppet/files/scap/dsh.scap-proxies
new file mode 100644
index 0000000..2fbb50c
--- /dev/null
+++ b/puppet/files/scap/dsh.scap-proxies
@@ -0,0 +1 @@
+localhost
diff --git a/puppet/files/scap/lint.php b/puppet/files/scap/lint.php
new file mode 100755
index 0000000..0cc87af
--- /dev/null
+++ b/puppet/files/scap/lint.php
@@ -0,0 +1,62 @@
+<?php
+
+if ( isset( $argv[1] ) ) {
+	$dir = $argv[1];
+	if ( !is_dir( $dir ) ) {
+		echo "Not a directory: $dir\n";
+		exit( 1 );
+	}
+} else {
+	$dir = '.';
+}
+
+if ( !check_dir( $dir ) ){
+	exit( 1 );
+} else {
+	exit( 0 );
+}
+
+function check_dir( $dir ) {
+	$handle = opendir( $dir );
+	if ( !$handle ) {
+		return true;
+	}
+	$success = true;
+	while ( false !== ( $fileName = readdir( $handle ) ) ) {
+		if ( substr( $fileName, 0, 1 ) == '.' ) {
+			continue;
+		}
+		if ( is_dir( "$dir/$fileName" ) ) {
+			$ret = check_dir( "$dir/$fileName" );
+		} elseif ( substr( $fileName, -4 ) == '.php' ) {
+			$ret = check_file( "$dir/$fileName" );
+		} else {
+			$ret = true;
+		}
+		$success = $success && $ret;
+	}
+	closedir( $handle );
+	return $success;
+}
+
+function check_file( $file ) {
+	static $okErrors = array(
+		'Redefining already defined constructor',
+		'Assigning the return value of new by reference is deprecated',
+	);
+	$errors = array();
+	parsekit_compile_file( $file, $errors, PARSEKIT_SIMPLE );
+	$ret = true;
+	if ( $errors ) {
+		foreach ( $errors as $error ) {
+			foreach ( $okErrors as $okError ) {
+				if ( substr( $error['errstr'], 0, strlen( $okError ) ) == $okError ) {
+					continue 2;
+				}
+			}
+			$ret = false;
+			print "Error in $file line {$error['lineno']}: {$error['errstr']}\n";
+		}
+	}
+	return $ret;
+}
diff --git a/puppet/files/scap/mw-deployment-vars.sh b/puppet/files/scap/mw-deployment-vars.sh
new file mode 100644
index 0000000..6bff3e3
--- /dev/null
+++ b/puppet/files/scap/mw-deployment-vars.sh
@@ -0,0 +1,10 @@
+# This file should be BASH / Python polyglot.
+MW_COMMON="/usr/local/apache/common-local"
+MW_COMMON_SOURCE="/a/common"
+MW_DBLISTS="/usr/local/apache/common-local"
+MW_DBLISTS_SOURCE="/a/common"
+MW_RSYNC_HOST="localhost"
+MW_DSH_ARGS=('-cM' '-g' 'mediawiki-installation' '-o' '-oSetupTimeout=30' '-F30')
+MW_RSYNC_ARGS=('-a' '--delete-delay' '--delay-updates' '--compress' '--delete' '--exclude=**/.svn/lock' '--exclude=**/.git/objects' '--exclude=**/.git/**/objects' '--exclude=**/cache/l10n/*.cdb' '--no-perms')
+MW_STATSD_HOST="127.0.0.1"
+MW_STATSD_PORT=2003
diff --git a/puppet/files/scap/mwscript b/puppet/files/scap/mwscript
new file mode 100755
index 0000000..bdb2733
--- /dev/null
+++ b/puppet/files/scap/mwscript
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Shell wrapper for the home version of multiversion/MWScript.php.
+# This script belongs in /usr/local/bin/ and should be in PATH.
+. /usr/local/lib/mw-deployment-vars.sh
+
+# Use the source version of the script if the source directory is present.
+# This only matters if the source dir is shared or when run on the deployment server.
+if [ -d "$MW_COMMON_SOURCE" ]; then
+	MW_COMMON_DIR_USE=$MW_COMMON_SOURCE
+else
+	MW_COMMON_DIR_USE=$MW_COMMON
+fi
+
+if groups | grep -Ewq 'sudo|wikidev|root'; then
+	sudo -u apache php "$MW_COMMON_DIR_USE/multiversion/MWScript.php" "$@"
+else
+	php "$MW_COMMON_DIR_USE/multiversion/MWScript.php" "$@"
+fi
diff --git a/puppet/files/scap/rsync.conf b/puppet/files/scap/rsync.conf
new file mode 100644
index 0000000..61622dc
--- /dev/null
+++ b/puppet/files/scap/rsync.conf
@@ -0,0 +1,17 @@
+# This file is being maintained by Puppet.
+# DO NOT EDIT
+uid = nobody
+gid = nogroup                                                                   use chroot = yes
+log format = %t %a %m %f %b
+syslog facility = local3
+timeout = 300
+address = 0.0.0.0
+
+[ common ]
+path            = /a/common
+read only       = true
+write only      = no
+list            = yes
+uid             = 0
+gid             = 0
+max connections = 0
diff --git a/puppet/files/scap/rsync.default b/puppet/files/scap/rsync.default
new file mode 100644
index 0000000..36e4d81
--- /dev/null
+++ b/puppet/files/scap/rsync.default
@@ -0,0 +1,5 @@
+RSYNC_ENABLE=true
+RSYNC_CONFIG_FILE=/etc/rsyncd.conf
+RSYNC_OPTS=''
+RSYNC_NICE=''
+
diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp
index 0633e31..8845331 100644
--- a/puppet/manifests/site.pp
+++ b/puppet/manifests/site.pp
@@ -13,3 +13,200 @@ import 'manifests.d/*.pp'
 node default {
     include role::mediawiki
 }
+
+node scap {
+    include role::generic
+    include ::php
+
+    group { 'mwdeploy':
+      ensure => present,
+    }
+
+    user { 'mwdeploy':
+      ensure     => present,
+      gid        => 'mwdeploy',
+      home       => '/var/lib/mwdeploy',
+      managehome => true,
+      shell      => '/bin/false',
+      system     => true,
+      require    => Group['mwdeploy'],
+    }
+
+    group { 'apache':
+      ensure => present,
+    }
+
+    user { 'apache':
+      ensure     => present,
+      gid        => 'apache',
+      home       => '/var/lib/apache',
+      managehome => true,
+      shell      => '/bin/false',
+      system     => true,
+      require    => Group['apache'],
+    }
+
+    group { 'l10nupdate':
+      ensure => present,
+    }
+
+    user { 'l10nupdate':
+      ensure     => present,
+      gid        => 'l10nupdate',
+      home       => '/var/lib/l10nupdate',
+      managehome => true,
+      shell      => '/bin/false',
+      system     => true,
+      require    => Group['l10nupdate'],
+    }
+
+    package { 'rsync':
+        ensure => present,
+    }
+
+    package { 'dsh':
+      ensure => present,
+    }
+
+    package { 'memcached':
+      ensure => present,
+    }
+
+    package { 'php5-parsekit':
+        ensure => present,
+    }
+
+    package { 'php5-memcached':
+      ensure => present,
+    }
+
+    file { '/etc/default/rsync':
+        ensure  => present,
+        mode    => 0444,
+        owner   => root,
+        group   => root,
+        source  => 'puppet:///files/scap/rsync.default',
+    }
+
+    file { '/etc/rsyncd.conf':
+        ensure  => present,
+        mode    => 0444,
+        owner   => root,
+        group   => root,
+        source  => 'puppet:///files/scap/rsync.conf',
+    }
+
+    service { 'rsync':
+        ensure    => running,
+        enable    => true,
+        subscribe => [
+            File['/etc/default/rsync'],
+            File['/etc/rsyncd.conf'],
+        ],
+    }
+
+    file { '/etc/dsh/group/scap-proxies':
+        ensure  => present,
+        source  => 'puppet:///files/scap/dsh.scap-proxies',
+        require => Package['dsh'],
+    }
+
+    file { '/etc/dsh/group/mediawiki-installation':
+        ensure  => present,
+        source  => 'puppet:///files/scap/dsh.mediawiki-installation',
+        require => Package['dsh'],
+    }
+
+    file { '/srv/scap':
+        ensure => link,
+        target => '/vagrant/scap',
+    }
+
+    file { '/usr/local/apache':
+        ensure => directory,
+    }
+
+    file { '/usr/local/apache/common-local':
+        ensure  => directory,
+        owner   => 'mwdeploy',
+        group   => 'mwdeploy',
+        mode    => 0755,
+    }
+
+    file { '/usr/local/apache/uncommon':
+        ensure  => directory,
+        owner   => 'mwdeploy',
+        group   => 'mwdeploy',
+        mode    => 0755,
+    }
+
+    file { '/usr/local/apache/common':
+        ensure  => link,
+        target  => '/usr/local/apache/common-local',
+    }
+
+    file { '/a':
+        ensure => directory,
+    }
+
+    file { '/a/common':
+        ensure  => link,
+        target  => '/vagrant/mediawiki-config',
+    }
+
+    file { '/usr/local/lib/mw-deployment-vars.sh':
+        ensure => present,
+        source => 'puppet:///files/scap/mw-deployment-vars.sh',
+    }
+
+    file { '/usr/local/bin/lint.php':
+        ensure => present,
+        source => 'puppet:///files/scap/lint.php',
+    }
+
+    file { '/usr/local/bin/mwscript':
+        ensure => present,
+        mode   => 0755,
+        source => 'puppet:///files/scap/mwscript',
+    }
+
+    file {
+        '/usr/local/bin/mw-update-l10n':
+            ensure => link,
+            target => '/srv/scap/bin/mw-update-l10n';
+        '/usr/local/bin/restart-twemproxy':
+            ensure => link,
+            target => '/srv/scap/bin/restart-twemproxy';
+        '/usr/local/bin/scap':
+            ensure => link,
+            target => '/srv/scap/bin/scap';
+        '/usr/local/bin/sync-common':
+            ensure => link,
+            target => '/srv/scap/bin/sync-common';
+        '/usr/local/bin/sync-dblist':
+            ensure => link,
+            target => '/srv/scap/bin/sync-dblist';
+        '/usr/local/bin/sync-dir':
+            ensure => link,
+            target => '/srv/scap/bin/sync-dir';
+        '/usr/local/bin/sync-docroot':
+            ensure => link,
+            target => '/srv/scap/bin/sync-docroot';
+        '/usr/local/bin/sync-file':
+            ensure => link,
+            target => '/srv/scap/bin/sync-file';
+        '/usr/local/bin/sync-wikiversions':
+            ensure => link,
+            target => '/srv/scap/bin/sync-wikiversions';
+        '/usr/local/bin/mwversionsinuse':
+            ensure => link,
+            target => '/srv/scap/bin/mwversionsinuse';
+        '/usr/local/bin/scap-rebuild-cdbs':
+            ensure => link,
+            target => '/srv/scap/bin/scap-rebuild-cdbs';
+        '/usr/local/bin/refreshCdbJsonFiles':
+            ensure => link,
+            target => '/srv/scap/bin/refreshCdbJsonFiles';
+    }
+
+}
-- 
2.0.0

I have mediawiki/tools/scap.git and operations/mediawiki-config.git checked out in the root of my vagrant project. Within mediawiki-config I have 2 checkouts of mediawiki/core.git on different tracking branches.