andrewmkhoury/How to identify DocumentNodeStore revision corruption.md

## format-revision.html
<html>
<head>
    <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
    <script>
        var Revision = function(rev) {
            	var dashIdx = rev.indexOf("-");
            	this.rev = rev;
            	this.timestamp = parseInt(rev.substring(1, dashIdx), 16);
            	this.counter = parseInt(rev.substring(dashIdx + 1, rev.indexOf("-", dashIdx + 1)), 16);
            	this.clusterId = parseInt(rev.substring(rev.lastIndexOf("-") + 1), 16);
        };

        Revision.prototype.toString = function () {
            return this.rev;
        };

        Revision.prototype.isNewerThan = function(other) {
            if (this.timestamp > other.timestamp) {
                return true;
            } else if (this.timestamp < other.timestamp) {
                return false;
            } else {
                return this.counter > other.counter;
            }
        };

        Revision.prototype.toReadableString = function () {
            return this.rev + " (" + this.asDate().toString() + ")"
        };

        Revision.prototype.asDate = function() {
            return new Date(this.timestamp);
        };

        Revision.prototype.getClusterId = function() {
            return this.clusterId;
        };

        formatRevision = function(rev) {
            return new Revision(rev).toReadableString();
        }
    </script>
</head>
<body>
    <form>
    	Revision 1: <input id="rev1" type="text" value="" onchange="">
    </form>
    <div id="datetime1">
    </div>
    <br>
    <form>
        Revision 2: <input id="rev2" type="text" value="" onchange="">
    </form>
    <div id="datetime2">
    </div>
    <br>
    <br>
    <div id="difference">

    </div>


    <br><br>
    Note: if the time between revisions is longer than a few minutes then there is likely data corruption.
<script>
    $('#rev1').each(function() {
       var elem = $(this);

       // Save current value of element
       elem.data('rev1', elem.val());

       // Look for changes in the value
       elem.bind("propertychange change click keyup input paste", function(event){
          // If value has changed...
          if (elem.data('rev1') != elem.val()) {
               // Updated stored value
               elem.data('rev1', elem.val());

               document.getElementById('datetime1').innerHTML= "Date and time of revision: " + formatRevision(this.value);
          }
       });
    });

    $('#rev2').each(function() {
       var elem = $(this);

       // Save current value of element
       elem.data('rev2', elem.val());

       // Look for changes in the value
       elem.bind("propertychange change click keyup input paste", function(event){
          // If value has changed...
          if (elem.data('rev2') != elem.val()) {
               // Updated stored value
               elem.data('rev2', elem.val());

               document.getElementById('datetime2').innerHTML= "Date and time of revision: " + formatRevision(this.value);

               ms = Math.abs(new Revision(document.getElementById('rev1').value).asDate() - new Revision(this.value).asDate());
               x = ms / 1000;
               seconds = Math.round(x % 60);
               x /= 60;
               minutes = Math.round(x % 60);
               x /= 60;
               hours = Math.round(x % 24);
               x /= 24;
               days = Math.round(x);
               document.getElementById('difference').innerHTML= "Difference in time between the revisions is " + ((days > 0)?days + " day" + ((days > 1)?"s":"") + ", " :"") + ((minutes > 0)?minutes + " min, " :"") + ((seconds > 0)?seconds + " s, " :"");
          }
       });
    });

</script>

</body>
</html>

## How to identify DocumentNodeStore revision corruption.md

      
    Raw
  

              How to identify DocumentNodeStore revision corruption.md
            
          
    If you find an error like this one in the logs (more of an example error below [1]) you may have a corrupt Oak revision:
Caused by: org.apache.jackrabbit.oak.plugins.document.DocumentStoreException: The node 4:/oak:index/event.job.topic/:index/replication was already added in revision
r14e64620028-0-1, before
r14e6463b2c0-0-1; document:
{_lastRev={r0-0-1=r14e64620028-0-1},
_deletedOnce=true,

To verify whether or not it is real corruption, do the following then save the format-revision.html to your system and open it in a browser.  Paste the two revisions into the corresponding text fields. You will get output like this:
Revision 1: r14e64620028-0-1
Date and time of revision: r14e64620028-0-1 (Mon Jul 06 2015 10:20:21 GMT-0700 (PDT))
Revision 2: r14e6463b2c0-0-1
Date and time of revision: r14e6463b2c0-0-1 (Mon Jul 06 2015 10:22:12 GMT-0700 (PDT))

In the above example, there is a time difference of roughly 2 minutes and 51 seconds, a time difference as long as this indicates that the node's revisions might be corrupt.  Minor time differences between two revisions could mean there was a conflict caused by concurrent modification of the same nodes.
To collect this data for post-analysis:


Upload this file https://s.apache.org/oak-mongo.js to the MongoDB server or use wget to download it
> wget https://s.apache.org/oak-mongo.js


Run the commands below on the server command line (from the same directory where oak-mongo.js file is located):
> mongo localhost:27017/aem-author --shell oak-mongo.js


Note: add authentication info to the command if required and change the db name in the above command to the aem mongo db that has the corrupt node.


Run this command in the mongo shell, make sure to replace "/path" with the path of the corrupt node (e.g. for the error above this is /oak:index/event.job.topic/:index/replication) and the db and host with the correct info matching the AEM mongo database.
> oak.printMongoExportCommand("/path", {db: "aem-author",host: "127.0.0.1"}); 
Note: change db: "aem-author" to the name of your aem database.


Then copy the output of the last command to your clipboard and exit the mongo shell
> exit


Paste output of the command in step 3 to the OS shell, it is a mongoexport command. Modify the command by removing "--out all-required-nodes.json" and add | gzip > --out all-required-nodes.json.gz to the end. Run the command (make sure the db, port and everything is correct and add credentials if required):
> mongoexport -h server1 --port 27017 --db aem-author --collection nodes --query '{$or:[{_id : /^4:p\/a\/b\//},{_id : /^3:p\/a\//},{_id : /^2:p\//},{_id:{$in:["2:/a/b","1:/a","0:/"]}}]}' | gzip > --out all-required-nodes.json.gz
This would generate a file all-required-nodes.json.gz file. That file would contain required information to analyze the corrupt node.


Note:
More recent Oak versions annotate the conflict revision with additional information to make it easier to identify a corruption.
Oak 1.4, 1.2.7, 1.0.23 introduced "(not yet visible)" next to the conflict revision when the other change was done on another cluster node, but the change is not yet visible on the local cluster node because background operations did not yet run. Exceptions with a conflict revision annotated with "(not yet visible)" are not repository corruptions. Such an exception message may look this like:
Caused by: org.apache.jackrabbit.oak.plugins.document.DocumentStoreException: The node 13:/oak:index/cqParentPath/:index/%2Fcontent%2Fbmw%2FmarketCA%2Fbmw_ca%2Fen_CA%2Fall-models%2F2-series%2Fcoupe%2Ftest/content/bmw/marketCA/bmw_ca/en_CA/all-models/2-series/coupe/test was already added in revision
r14fcc91e24e-0-5 (not yet visible), before
r14fcc91e33f-0-6; document:

[1]
Caused by: org.apache.jackrabbit.oak.plugins.document.DocumentStoreException: The node 11:/oak:index/nodetype/:index/slingevent%3AJob/var/eventing/jobs/assigned/3d5b00fc-cad1-4730-b715-ee359d24c1ff/com.day.cq.replication.job.flush_ds04/2016 was already added in revision
r1536712bdd2-0-3, before
r1537fbedf1c-0-1; document:
{_lastRev={r0-0-7=r1532e9a5205-0-7, r0-0-3=r1536712bdd2-0-3, r0-0-2=r15364382500-0-2, r0-0-1=r1536712b278-0-1},
_prev={r15367106fd0-0-1=r15366e502d8-0-1/0, r15367103a49-0-3=r15363bcf7c6-0-3/1, r15366e4f0f1-0-1=r153669c4918-0-1/0, r153669c1872-0-1=r153667d5134-0-1/0, r153667d3d16-0-1=r1536657d92c-0-1/0, r1536655ab02-0-1=r1536638e30d-0-1/0, r1536638d0ff-0-1=r153623af9a0-0-1/1, r1536424a1b5-0-2=r1536415d857-0-2/0, r1536415b73b-0-2=r15364113aa1-0-2/0, r15364112896-0-2=r1535f2996a6-0-2/0, r15363bbecb6-0-3=r1535f3a6b39-0-3/1, r153623978c2-1-1=r1535f2a1690-0-1/1, r1535f2985df-0-2=r1535f10df32-0-2/0, r1535f10c3f4-0-2=r1535f0cb238-0-2/0, r1535f0ca4d5-0-2=r1535f08ac40-0-2/0, r1535f089cf2-0-2=r1535f04795a-0-2/0, r1535f046470-0-2=r1535f00a3af-0-2/0, r1535f008591-0-2=r15354b4182c-0-2/1},
...
	<html>
	<head>
	<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
	<script>
	var Revision = function(rev) {
	var dashIdx = rev.indexOf("-");
	this.rev = rev;
	this.timestamp = parseInt(rev.substring(1, dashIdx), 16);
	this.counter = parseInt(rev.substring(dashIdx + 1, rev.indexOf("-", dashIdx + 1)), 16);
	this.clusterId = parseInt(rev.substring(rev.lastIndexOf("-") + 1), 16);
	};

	Revision.prototype.toString = function () {
	return this.rev;
	};

	Revision.prototype.isNewerThan = function(other) {
	if (this.timestamp > other.timestamp) {
	return true;
	} else if (this.timestamp < other.timestamp) {
	return false;
	} else {
	return this.counter > other.counter;
	}
	};

	Revision.prototype.toReadableString = function () {
	return this.rev + " (" + this.asDate().toString() + ")"
	};

	Revision.prototype.asDate = function() {
	return new Date(this.timestamp);
	};

	Revision.prototype.getClusterId = function() {
	return this.clusterId;
	};

	formatRevision = function(rev) {
	return new Revision(rev).toReadableString();
	}
	</script>
	</head>
	<body>
	<form>
	Revision 1: <input id="rev1" type="text" value="" onchange="">
	</form>
	<div id="datetime1">
	</div>
	<br>
	<form>
	Revision 2: <input id="rev2" type="text" value="" onchange="">
	</form>
	<div id="datetime2">
	</div>
	<br>
	<br>
	<div id="difference">

	</div>


	<br><br>
	Note: if the time between revisions is longer than a few minutes then there is likely data corruption.
	<script>
	$('#rev1').each(function() {
	var elem = $(this);

	// Save current value of element
	elem.data('rev1', elem.val());

	// Look for changes in the value
	elem.bind("propertychange change click keyup input paste", function(event){
	// If value has changed...
	if (elem.data('rev1') != elem.val()) {
	// Updated stored value
	elem.data('rev1', elem.val());

	document.getElementById('datetime1').innerHTML= "Date and time of revision: " + formatRevision(this.value);
	}
	});
	});

	$('#rev2').each(function() {
	var elem = $(this);

	// Save current value of element
	elem.data('rev2', elem.val());

	// Look for changes in the value
	elem.bind("propertychange change click keyup input paste", function(event){
	// If value has changed...
	if (elem.data('rev2') != elem.val()) {
	// Updated stored value
	elem.data('rev2', elem.val());

	document.getElementById('datetime2').innerHTML= "Date and time of revision: " + formatRevision(this.value);

	ms = Math.abs(new Revision(document.getElementById('rev1').value).asDate() - new Revision(this.value).asDate());
	x = ms / 1000;
	seconds = Math.round(x % 60);
	x /= 60;
	minutes = Math.round(x % 60);
	x /= 60;
	hours = Math.round(x % 24);
	x /= 24;
	days = Math.round(x);
	document.getElementById('difference').innerHTML= "Difference in time between the revisions is " + ((days > 0)?days + " day" + ((days > 1)?"s":"") + ", " :"") + ((minutes > 0)?minutes + " min, " :"") + ((seconds > 0)?seconds + " s, " :"");
	}
	});
	});

	</script>

	</body>
	</html>