Oak count node check in script console - use while AEM / Oak is running

1 Instructions - Find Oak Corruptions.md

NOTE: Steps moved here instead https://github.com/cqsupport/fix-instructions/tree/master/count-nodes

---

1. Go to http://host:port/system/console/bundles and install these two bundles
   • http://apache.mirrors.lucidnetworks.net//felix/org.apache.felix.webconsole.plugins.scriptconsole-1.0.2.jar
   • https://repo1.maven.org/maven2/org/codehaus/groovy/groovy-all/2.4.9/groovy-all-2.4.9.jar
2. Go to http://host/system/console/configMgr/org.apache.sling.jcr.base.internal.LoginAdminWhitelist
3. Add org.apache.felix.webconsole.plugins.scriptconsole to "Whitelist regexp" and save
4. After the two bundles fully install then go to http://host:port/system/console/slinglog
5. Click "Add New Logger" and set log level to "Info", "Log File" to "logs/countnodes.log", and "Logger" to "countNodes.groovy"
6. Save the log file config
7. Go to http://host:port/system/console/sc
8. Select "Groovy" as the language
9. Copy/Paste the contents of countNodes-*.groovy script below which matches your oak version to the script console
10. Click "Execute"
11. The output will go to crx-quicktstart/logs/countnodes.log

Side note: The scripts below are an adaptation of this script updated to work in the Felix Script console: https://gist.github.com/stillalex/06303f8cc1d3780d3eab4c72575883ae

countNodes-oak14andOlder.groovy

import java.io.InputStream;
import java.util.concurrent.atomic.AtomicInteger
import org.apache.jackrabbit.oak.api.Type
import org.apache.jackrabbit.oak.plugins.segment.SegmentBlob
import org.apache.jackrabbit.oak.spi.state.NodeState

def countNodes(NodeState n, deep = false, String path = "/", Integer flush = 50000, AtomicInteger count = new AtomicInteger(0), AtomicInteger binaries = new AtomicInteger(0), root = true) {
  if(root) {
    out.println "Counting nodes in tree ${path}"
  }

  cnt = count.incrementAndGet()
  if (cnt % flush == 0) out.println("  " + cnt)
  def propname = "";
  try {
    try {

        for(prop in n.getProperties()) {
          propname = prop.getName();
          if(prop.getType() == Type.BINARY || prop.getType() == Type.BINARIES) {
            for(b in prop.getValue(Type.BINARIES)) {
              binaries.incrementAndGet()
              if(deep) {
                  InputStream s = b.getNewStream();
                  try {
                    byte[] buffer = new byte[1024];
                    int l = s.read(buffer, 0, buffer.length);
                  } finally {
                    s.close();
                  }
              } else if(b instanceof SegmentBlob) {
                if(!((SegmentBlob)b).isExternal()) {
                  b.length()
                }
              } else {
                b.length()
              }
            }
          } else {
            if(prop.isArray()) {
                for(sf in prop.getValue(prop.getType())) {
                    // do nothing - we just need to read all values
                }
            } else {
                prop.getValue(prop.getType());
            }

          }
        }
    } catch(ex) {
        out.println "warning unable to read node ${path}@" + propname + " : " + ex.getMessage()
    }
    propname = "";

    for(child in n.getChildNodeEntries()) {
      countNodes(child.getNodeState(), deep, path + child.getName() + "/", flush, count, binaries, false)
    }
  } catch(e) {
    out.println "warning unable to read node ${path} : " + e.getMessage()
  }

  if(root) {
    out.println "Total nodes in tree ${path}: ${cnt}"
    out.println "Total binaries in tree ${path}: ${binaries.get()}"
  }

  return cnt
}

def countNodes(session) {
    def nstore = session.getRootNode().sessionDelegate.root.store
    def rs = nstore.root
    out.println("Running node counter")
    countNodes(rs)
    out.println("Done")
    null
}

def countNodes() {
    def repo = osgi.getService(org.apache.sling.jcr.api.SlingRepository)
    def session = repo.loginAdministrative(null)
    try {
        countNodes(session)
    } finally {
        session.logout()
    }
}

countNodes()

countNodes-oak16andLater.groovy

//Adaptation of @stillalex's script from here https://gist.github.com/stillalex/06303f8cc1d3780d3eab4c72575883ae
//This version works with Oak 1.6 and later versions
import java.io.InputStream;
import java.util.concurrent.atomic.AtomicInteger
import org.apache.jackrabbit.oak.api.Type
import org.apache.jackrabbit.oak.spi.state.NodeState
import org.apache.jackrabbit.oak.spi.state.NodeStore
import org.apache.jackrabbit.oak.commons.PathUtils
import com.google.common.collect.Lists
import java.util.List

org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger("countNodes.groovy");

def countNodes(NodeState n, deep = false, String path = "/", Integer flush = 1000, AtomicInteger count = new AtomicInteger(0), AtomicInteger binaries = new AtomicInteger(0), root = true) {
  org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger("countNodes.groovy");

  if(root) {
    log.info("Counting nodes in tree ${path}");
  }
 
  cnt = count.incrementAndGet()
  if (cnt % flush == 0) log.info("  " + cnt)

  try {
      for(prop in n.getProperties()) {
        try {
          if(prop.getType() == Type.BINARIES) {
              for(b in prop.getValue(Type.BINARIES)) {
                binaries.incrementAndGet()
                if(deep) {
                    InputStream s = b.getNewStream();
                    try {
                      byte[] buffer = new byte[1024];
                      int l = s.read(buffer, 0, buffer.length);
                    } finally {
                      s.close();
                    }
                } else {
                    b.length();
                }
                binaries.incrementAndGet();
              }
        } else if(prop.getType() == Type.BINARY) {
            def b = prop.getValue(Type.BINARY);
            if(deep) {
                InputStream s = b.getNewStream();
                try {
                  byte[] buffer = new byte[1024];
                  int l = s.read(buffer, 0, buffer.length);
                } finally {
                  s.close();
                }
            } else {
                b.length();
            }
            binaries.incrementAndGet();
        } else {
            // Check regular properties for missing segments 
            if(prop.isArray()) {
                for(sf in prop.getValue(prop.getType())) {
                    // do nothing - we just need to read all values
                }
            } else {
                prop.getValue(prop.getType());
            }
        }
      } catch(e) {
          log.error("warning unable to read node properties ${path} ${prop.name}: " + e.getMessage())
        //org.codehaus.groovy.runtime.StackTraceUtils.printSanitizedStackTrace(e, out)
      }
    }

    try {
      for(child in n.getChildNodeEntries()) {
        try {
          countNodes(child.getNodeState(), deep, path + "/" + child.getName(), flush, count, binaries, false)
        } catch(e) {
          log.error("warning unable to read child node ${path} : " + e.getMessage())
	      //org.codehaus.groovy.runtime.StackTraceUtils.printSanitizedStackTrace(e, out)
        }
      }
    } catch(e) {
      log.error("warning unable to read child entries ${path} : " + e.getMessage())
      //org.codehaus.groovy.runtime.StackTraceUtils.printSanitizedStackTrace(e, out)
    }
  } catch(e) {
    log.error("warning unable to read node ${path} : " + e.getMessage())
    //org.codehaus.groovy.runtime.StackTraceUtils.printSanitizedStackTrace(e, out)
  }

  if(root) {
    log.info("Total nodes in tree ${path}: ${cnt}");
    log.info("Total binaries in tree ${path}: ${binaries.get()}");
  }

  return cnt
}

def countNodes(session, path, deep) {
    NodeStore nstore = session.getRootNode().sessionDelegate.root.store
    def rs = nstore.root
    def rnb = rs.builder()
    def nb = rnb;
    List<String> elements = Lists.newArrayList();
    PathUtils.elements(path).each{String element ->
        if (PathUtils.denotesParent(element)) {
            if (!elements.isEmpty()) {
                elements.remove(elements.size() - 1);
            }
        } else if (!PathUtils.denotesCurrent(element)) {
            elements.add(element);
        }
    }

    elements.each {
      if(it.size() > 0) {
        nb = nb.getChildNode(it)
      }
    }
    countNodes(nb.getNodeState(), deep, path)
}

def countNodes(path) {
	def repo = osgi.getService(org.apache.sling.jcr.api.SlingRepository)
	def session = repo.loginAdministrative(null)
	try {
	    countNodes(session, path, true)
	} finally {
	    session.logout()
	}
}

log.info("Running node counter");
// Or only check the async oak indexes
t1 = Thread.start("countNodes /oak:index",{countNodes("/oak:index")})
t2 = Thread.start("countNodes /content", {countNodes("/content")})
t3 = Thread.start("countNodes /var", {countNodes("/var")})
t4 = Thread.start("countNodes /etc", {countNodes("/etc")})
t5 = Thread.start("countNodes /tmp", {countNodes("/tmp")})
t6 = Thread.start("countNodes /conf", {countNodes("/conf")})
t7 = Thread.start("countNodes /jcr:system", {countNodes("/jcr:system")})
t8 = Thread.start("countNodes /apps", {countNodes("/apps")})
t9 = Thread.start("countNodes /libs", {countNodes("/libs")})
t10 = Thread.start("countNodes /home", {countNodes("/home")})

log.info("Done starting countNodes threads");

t1.join();
t2.join();
t3.join();
t4.join();
t5.join();
t6.join();
t7.join();
t8.join();
t9.join();
t10.join();

log.info("Done running countNodes");

null

countNodes-oakRun14andolder.groovy

import java.io.InputStream;
import java.util.concurrent.atomic.AtomicInteger
import org.apache.jackrabbit.oak.api.Type
import org.apache.jackrabbit.oak.plugins.segment.SegmentBlob
import org.apache.jackrabbit.oak.spi.state.NodeState

def countNodes(NodeState n, deep = false, String path = "/", Integer flush = 50000, AtomicInteger count = new AtomicInteger(0), AtomicInteger binaries = new AtomicInteger(0), root = true) {
  if(root) {
    println "Counting nodes in tree ${path}"
  }

  cnt = count.incrementAndGet()
  if (cnt % flush == 0) println("  " + cnt)
  def propname = "";
  try {
    try {

        for(prop in n.getProperties()) {
          propname = prop.getName();
          if(prop.getType() == Type.BINARY || prop.getType() == Type.BINARIES) {
            for(b in prop.getValue(Type.BINARIES)) {
              binaries.incrementAndGet()
              if(deep) {
                  InputStream s = b.getNewStream();
                  try {
                    byte[] buffer = new byte[1024];
                    int l = s.read(buffer, 0, buffer.length);
                  } finally {
                    s.close();
                  }
              } else if(b instanceof SegmentBlob) {
                if(!((SegmentBlob)b).isExternal()) {
                  b.length()
                }
              } else {
                b.length()
              }
            }
          } else {
            if(prop.isArray()) {
                for(sf in prop.getValue(prop.getType())) {
                    // do nothing - we just need to read all values
                }
            } else {
                prop.getValue(prop.getType());
            }

          }
        }
    } catch(ex) {
        println "warning unable to read node ${path}@" + propname + " : " + ex.getMessage()
    }
    propname = "";

    for(child in n.getChildNodeEntries()) {
      countNodes(child.getNodeState(), deep, path + child.getName() + "/", flush, count, binaries, false)
    }
  } catch(e) {
    println "warning unable to read node ${path} : " + e.getMessage()
  }

  if(root) {
    println "Total nodes in tree ${path}: ${cnt}"
    println "Total binaries in tree ${path}: ${binaries.get()}"
  }

  return cnt
}

countNodes(session.workingNode)

countNodes-oakRun64later.groovy

//Adaptation of @stillalex's script from here https://gist.github.com/stillalex/06303f8cc1d3780d3eab4c72575883ae
//This version works with Oak 1.6 and later versions
import java.io.InputStream;
import java.util.concurrent.atomic.AtomicInteger
import org.apache.jackrabbit.oak.api.Type
import org.apache.jackrabbit.oak.spi.state.NodeState
import org.apache.jackrabbit.oak.spi.state.NodeStore

def countNodes(n, deep = false, String path = "/", Integer flush = 100000, AtomicInteger count = new AtomicInteger(0), AtomicInteger binaries = new AtomicInteger(0), root = true) {
  if(root) {
    println "Counting nodes in tree ${path}"
  }

  cnt = count.incrementAndGet()
  if (cnt % flush == 0) println("  " + cnt)

  try {
    try {
      for(prop in n.getProperties()) {
        if(prop.getType() == Type.BINARY || prop.getType() == Type.BINARIES) {
            for(b in prop.getValue(Type.BINARIES)) {
              binaries.incrementAndGet()
              if(b instanceof SegmentBlob) {
                if(!((SegmentBlob)b).isExternal()) {
                  b.length()
                }
              }
            }
        } else if(prop.getType() == Type.BINARY) {
          def b = prop.getValue(Type.BINARY);
          binaries.incrementAndGet()
          if(b instanceof SegmentBlob) {
            if(!((SegmentBlob)b).isExternal()) {
              b.length()
            }
          }
          binaries.incrementAndGet();
        } else {
            // Check regular properties for missing segments 
            if(prop.isArray()) {
                for(sf in prop.getValue(prop.getType())) {
                    // do nothing - we just need to read all values
                }
            } else {
                prop.getValue(prop.getType());
            }
        }
      }
    } catch(e) {
      println "warning unable to read node properties ${path} : " + e.getMessage()
      org.codehaus.groovy.runtime.StackTraceUtils.printSanitizedStackTrace(e)
    }
    try {
      for(child in n.getChildNodeEntries()) {
        try {
          countNodes(child.getNodeState(), deep, path + child.getName() + "/", flush, count, binaries, false)
        } catch(e) {
          println "warning unable to read child node ${path} : " + e.getMessage()
	  org.codehaus.groovy.runtime.StackTraceUtils.printSanitizedStackTrace(e)
        }
      }
    } catch(e) {
      println "warning unable to read child entries ${path} : " + e.getMessage()
      org.codehaus.groovy.runtime.StackTraceUtils.printSanitizedStackTrace(e)
    }
  } catch(e) {
    println "warning unable to read node ${path} : " + e.getMessage()
    org.codehaus.groovy.runtime.StackTraceUtils.printSanitizedStackTrace(e)
  }

  if(root) {
    println "Total nodes in tree ${path}: ${cnt}"
    println "Total binaries in tree ${path}: ${binaries.get()}"
  }

  return cnt
}

The file "countNodes-oakRun64later.groovy" seems to be missing this line at the end:
countNodes(session.workingNode)

@johnb4 Go ahead and update it here, I moved this documentation https://github.com/cqsupport/fix-instructions/tree/master/count-nodes