README.md

This is a test of an idea to use local persistent volumes and initContainers for rebuilding the MW l10n cache during deployment instead of including the caches in the image itself.

Run setup.sh to get started

setup.sh

Running setup.sh should get you a two-worker node k8s cluster running locally along with a local image registry and a built multiversion image.

Create the deployment

kubectl apply -f mediawiki.yaml

Running this command should create a deployment for MediaWiki of 6 pods that uses an init container to rebuild the l10n cache.

Verify that only one rebuild process is running at once on each node

The init container uses a file lock to ensure that only one rebuild process is running on each node which should work properly on the local storage backed persistent volume. Verify this during the deployment by attaching to a node docker container and running top (you can see both processes even though they're separate nodes because this is using kind and the nodes are within the same VM).

docker exec -it test-local-pvs-worker top

blubber.yaml

version: v4
variants:
  multiversion:
    base: docker-registry.wikimedia.org/php7.2-fpm
    lives:
      in: /srv/mediawiki
    runs:
      # match base image www-data uid/gid
      uid: 33
      gid: 33
    builder:
      requirements:
        - .
        - from: docker-registry.wikimedia.org/wikimedia/mediawiki:wmf-1.37.0-wmf.14
          source: /srv/mediawiki
          destination: /srv/mediawiki/php-1.37.0-wmf.14
      command:
        - sh
        - -c
        - >-
          touch private/PrivateSettings.php &&
          .pipeline/wmf-publish/build prepare-mw-versions &&
          .pipeline/wmf-publish/build merge-message-file-list

cluster.yaml

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
  - role: control-plane
  - role: worker
  - role: worker
containerdConfigPatches:
  - |-
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
      endpoint = ["http://kind-registry:5000"]

mediawiki.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: mediawiki
spec:
  replicas: 6
  selector:
    matchLabels:
      app: mediawiki
  template:
    metadata:
      labels:
        app: mediawiki
    spec:
      securityContext:
        fsGroup: 33
        fsGroupChangePolicy: OnRootMismatch
      initContainers:
        - name: l10n-rebuild
          image: localhost:5000/mediawiki-multiversion:pv-test
          imagePullPolicy: Always
          volumeMounts:
            - name: l10n
              mountPath: /srv/mediawiki/php-1.37.0-wmf.14/cache
          command:
            - bash
            - -c
            - >-
              (
                flock -x 666;
                .pipeline/wmf-publish/build rebuild-l10n-cache
              ) 666> php-1.37.0-wmf.14/cache/l10n.lock

      containers:
        - name: mediawiki
          image: localhost:5000/mediawiki-multiversion:pv-test
          command: [sh, -c, 'while sleep 1; do true; done']
          volumeMounts:
            - name: l10n
              mountPath: /srv/mediawiki/php-1.37.0-wmf.14/cache
      volumes:
        - name: l10n
          persistentVolumeClaim:
            claimName: l10n-cache-claim

pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: l10n-cache-pv
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  accessModes:
  - ReadWriteMany
  - ReadOnlyMany
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage
  local:
    path: /mnt/l10n-cache
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - test-local-pvs-worker
          - test-local-pvs-worker2

pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: l10n-cache-claim
spec:
  storageClassName: local-storage
  accessModes:
    - ReadWriteMany
    - ReadOnlyMany
  resources:
    requests:
      storage: 5Gi

registry-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: local-registry-hosting
  namespace: kube-public
data:
  localRegistryHosting.v1: |
    host: "localhost:5000"
    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"

setup.sh

#!/bin/bash

set -euo pipefail

if ! command -v kind &> /dev/null; then
  echo "kind is not installed on your system"
  echo "If you have go tools available, you may install it via:"
  echo "GO111MODULE=\"on\" go get sigs.k8s.io/kind@v0.11.1"
  echo "(see https://kind.sigs.k8s.io/)"
  exit 1
fi

echo "Creating two-worker-node cluster using kind"
kind create cluster --name test-local-pvs --config cluster.yaml
kubectl config use-context kind-test-local-pvs

echo "Starting up a local docker registry and making it known to the cluster"
docker run \
    -d --restart=always \
    -p "127.0.0.1:5000:5000" \
    --name kind-registry \
    registry:2

docker network connect kind kind-registry
kubectl apply -f registry-configmap.yaml

echo "Building and registering a multiversion MW image"
git clone https://gerrit.wikimedia.org/r/operations/mediawiki-config
git -C mediawiki-config checkout 9d38a3e0d5d7d9d448e3666c75f07341d9a471be

curl \
  -s https://blubberoid.wikimedia.org/v1/multiversion \
  -H 'content-type: application/yaml' \
  --data-binary @blubber.yaml | \
  docker build \
    -t localhost:5000/mediawiki-multiversion:pv-test \
    -f - mediawiki-config

docker push localhost:5000/mediawiki-multiversion:pv-test

# Create a dir on each node for the local storage pv
echo "Provisioning directories for local storage PersistentVolume"
docker ps --filter 'label=io.x-k8s.kind.role=worker' --format '{{.ID}}' | while read id; do
  docker exec $id mkdir -p /mnt/l10n-cache
done

echo "Installing PV related resources"
kubectl apply \
  -f storageclass.yaml \
  -f pv.yaml \
  -f pvc.yaml

echo "Done. You may now deploy MediaWiki via:"
echo "kubectl apply -f mediawiki.yaml"

storageclass.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: local-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer