1-Oak SegmentNotFoundException repair scenario.md

The objective here is to find all SegmentNotFoundException type corruption and remove it (as we don't have a valid backup)

Before running any of the below steps we created an index.json file (see the file below) which contains only the index definitions of the corrupted indexes.

1. This generates the file indexing-results\index-definities.json

java -Xmx8g -jar oak-run-1.8.12.jar index --fds-path=crx-quickstart\repository\datastore crx-quickstart\repository\segmentstore --index-definitions

2. From that file we copied only the definitions we needed and created index.json (file attached here )

Steps to remove corrupt nodes and reindex:

1. Download the oak-run version matching your oak version: https://repo1.maven.org/maven2/org/apache/jackrabbit/oak-run/

2. Upload the file rmCorruptIndexes.groovy to the server

3. Run the oak-run console tool to open a shell:

   java -Xmx8g -jar oak-run-1.*.jar console crx-quickstart/repositor/segmentstore --read-write
   

4. Run this command in the oak-run shell to load the groovy script and delete the bad indexes:

   :load rmCorruptIndexes.groovy
   

5. Remove all checkpoints (to bypass corruption existing between checkpoint revision and head revision of repository)

   java -Xmx8g -jar oak-run-1.*.jar checkpoints crx-quickstart/repositor/segmentstore --rm-all
   

6. Run an offline compaction per these instructions https://gist.github.com/andrewmkhoury/0b1fe4d8b619178ff87b

   THIS IS FAILING WITH A SegmentNotFoundException - see here

7. Run the reindex.bat script below to reindex these indexes /oak:index/uuid,/oak:index/repMembers,/oak:index/reference,/oak:index/acPrincipalName,/oak:index/nodetype Note that the last command uses oak-upgrade to copy over the indexing results because the index import feature of the offline indexing tool is failing. The last part of the script opens the oak-run console, we found that this is necessary for AEM to function after the offline indexing.

   THIS IS FAILING WITH A SegmentNotFoundException - see here

aem-setup.bash

#!/bin/bash

MEM_AVAILABLE=$((($( grep -i memtotal /proc/meminfo | awk '{print $2}')/1048576)-4))
if [[ "$MEM_AVAILABLE" -lt 0 ]]; then
    logger -p local0.warn -t AEM "WARNING: AEM is starting with 2GB (or less) of RAM"
    MEM_AVAILABLE=$((($( grep -i memtotal /proc/meminfo | awk '{print $2}')/1048576)/2))
fi

XMX_TOTAL=$(echo -e "${MEM_AVAILABLE}\n0.4\n*\n1/\np" | /usr/bin/dc)

if [[ "$XMX_TOTAL" -lt 1 ]]; then
    logger -p local0.warn -t AEM "WARNING: AEM is starting with XMX set to less than 2GB of RAM"
    XMX_TOTAL=1
fi

ENV_TAG="$(su - aem -c 'aws ec2 describe-tags --filters Name=resource-id,Values="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" | jq -r '\'' .Tags[] | select(.Key | contains("env")).Value '\'' ')"
INSTANCE_TAG="$(su - aem -c 'aws ec2 describe-tags --filters Name=resource-id,Values="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" | jq -r '\'' .Tags[] | select(.Key | contains("instance")).Value '\'' ')
MOTD_NAME="$(echo "${ENV_TAG}" | awk '{print toupper(substr($0,1,1)) tolower(substr($0,2)) }')"
sudo sed -i -e "s@Stage\|UAT\|Dev\|QA\|Prod\|Int\|Test@${MOTD_NAME}@i" /etc/motd

if [[ "${ENV_TAG}" = *"dev"* ]]; then
    SAMPLECONTENT="samplecontent"
else
    SAMPLECONTENT="nosamplecontent"
fi

S3_VOLUME_FREE="$(df -B1 /opt/aem/s3-cache | tail -1 | awk '{print $4}')"
S3_CACHE_SIZE="$(echo -e "${S3_VOLUME_FREE}\n0.97*\n1/\np" | dc)"
S3_BUCKET=""

if [[ -z "${S3_BUCKET}" ]]; then
    echo "YOU MUST SET AN S3 BUCKET"
    exit 1
fi

cat << EOF > /etc/aem/aem.conf
CQ_PORT='4502'
CQ_HOST='0.0.0.0'
CQ_RUNMODE="author,sixfive,${SAMPLECONTENT},${ENV_TAG:-dev}"
CQ_JVM_OPTS="-server \
-Djava.awt.headless=true \
-Djava.io.tmpdir=/opt/aem/tmp \
-Djava.net.preferIPv4Stack=true \
-Doak.fastQuerySize=true \
-XX:+UseConcMarkSweepGC \
-XX:MaxDirectMemorySize=${MEM_AVAILABLE}g \
-XX:MaxMetaspaceSize=1g \
-Xmx${XMX_TOTAL}g \
-XX:+HeapDumpOnOutOfMemoryError \
-XX:HeapDumpPath=/opt/aem/logs/oom.hprof \
-XX:+PrintGCDateStamps -verbose:gc \
-XX:+PrintGCDetails \
-XX:+UseGCLogFileRotation \
-XX:NumberOfGCLogFiles=5 \
-XX:GCLogFileSize=200M \
-Xloggc:/opt/aem/author/crx-quickstart/logs/gc.log"
EOF

mkdir -p /opt/aem/author/crx-quickstart/install

cat << EOT > /opt/aem/author/crx-quickstart/install/org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStore.config
accessKey="AKIAI4IFED4FZEIXJ54A"
secretKey="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"
s3Bucket="${S3_BUCKET}"
s3Region="us-standard"
minRecordLength="16384"
maxConnections="70"
maxCachedBinarySize="32768"
uploadThreads="20"
maxErrorRetry="10"
socketTimeout="120000"
connectionTimeout="120000"
writeThreads="30"
cacheSize="${S3_CACHE_SIZE}"
path="/opt/aem/s3-cache"
EOT

chown -Rv aem:users /opt/aem

rmNode-2019.07.18.groovy

import org.apache.jackrabbit.oak.spi.commit.CommitInfo
import org.apache.jackrabbit.oak.spi.commit.EmptyHook
import org.apache.jackrabbit.oak.spi.state.NodeStore
import org.apache.jackrabbit.oak.commons.PathUtils

def rmNode(def session, String path) {
    println "Removing node ${path}"

    NodeStore ns = session.store
    def nb = ns.root.builder()

    def aBuilder = nb
    for(p in PathUtils.elements(path)) {  aBuilder = aBuilder.getChildNode(p) }

    if(aBuilder.exists()) {
        rm = aBuilder.remove()
        ns.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY)
        return rm
    } else {
        println "Node ${path} doesn't exist"
        return false
    }
}

rmNode(session, "/oak:index/uuid/:index")
rmNode(session, "/oak:index/repMembers/:index")
rmNode(session, "/oak:index/reference/:weakreferences")
rmNode(session, "/oak:index/reference/:references")
rmNode(session, "/oak:index/acPrincipalName/:index")
rmNode(session, "/oak:index/nodetype/:index")
rmNode(session, "/oak:index/principalName/:index")
rmNode(session, "/oak:index/authorizableId/:index")
rmNode(session, "/oak:index/externalId/:index")
rmNode(session, "/oak:index/externalPrincipalNames/:index")
rmNode(session, "/content/dam/2016/1-27-2016/svc-cmingest/CWD_28EB5723-7A79-71BA-E053-CB02630A567D.jpg")

rmNode.groovy

import org.apache.jackrabbit.oak.spi.commit.CommitInfo
import org.apache.jackrabbit.oak.spi.commit.EmptyHook
import org.apache.jackrabbit.oak.spi.state.NodeStore
import org.apache.jackrabbit.oak.commons.PathUtils

def rmNode(def session, String path) {
    println "Removing node ${path}"

    NodeStore ns = session.store
    def nb = ns.root.builder()

    def aBuilder = nb
    for(p in PathUtils.elements(path)) {  aBuilder = aBuilder.getChildNode(p) }

    if(aBuilder.exists()) {
        rm = aBuilder.remove()
        ns.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY)
        return rm
    } else {
        println "Node ${path} doesn't exist"
        return false
    }
}