Skip to content

Instantly share code, notes, and snippets.

@littleskunk
Last active February 27, 2018 08:51
Show Gist options
  • Save littleskunk/9f526cdbfa99a5891d097558af77e510 to your computer and use it in GitHub Desktop.
Save littleskunk/9f526cdbfa99a5891d097558af77e510 to your computer and use it in GitHub Desktop.
// The shard reaper will only delete expired contract inclusive shards.
// He will not detect shards with a missing contract
// This script will delete the lost shards
// How to run it
// Requires nodejs LTS, git, python2
// Open command line and move to the folder where this script is stored
// Execute "npm install storj-lib"
// Execute "node Storj_Farmer_Delete_Lost_Shards.js"
var async = require('async');
var storj = require('storj-lib');
var stream = require('readable-stream');
var persistence = new storj.EmbeddedStorageAdapter('insert your storage location here');
var manager = new storj.StorageManager(persistence);
var StorageAdapter = require('storj-lib/lib/storage/adapter');
var StorageItem = require('storj-lib/lib/storage/item');
var rstream = manager._storage.createReadStream();
var list = [];
var buckets = [];
for(var i = 0; i <= 255; i++){
buckets.push(i);
}
console.log('Reading contracts');
rstream.on('data', function(item) {
rstream.pause();
list.push(item.fskey || item.hash);
rstream.resume();
});
rstream.on('end', function() {
console.log('Reading contracts finished');
async.forEachLimit(buckets, 1, function (i, callback){
console.log('Reading bucket %s', i);
persistence._fs.list(i, (err, keys) => {
if (keys) {
keys.forEach((result) => {
if (list.indexOf(result.baseKey) == -1) {
persistence._fs.unlink(result.baseKey, function(/* err */) {
console.log('Deleting lost shard: %s', result.baseKey);
});
}
});
}
persistence._fs._getSbucketAtIndex(i).close();
callback();
});
}, function(err) {
console.log('flushing shards, some buckets will be inaccessible');
manager._storage.flush(function(err) {
/* istanbul ignore if */
if (err) {
console.log('problem while flushing shards, %s', err.message);
}
console.log('flushing shards finished');
process.exit();
});
});
});
// How to run it
// Requires nodejs LTS, git, python2
// Open command line and move to the folder where this script is stored
// Execute "npm install storj-lib"
// Execute "node Storj_Farmer_Manuall_Reaper.js"
var storj = require('storj-lib');
// insert your storage location here.
// on Windows you need double backslash. C:\\storjshare\\whatever
var persistence = new storj.EmbeddedStorageAdapter('insert your storage location here');
var manager = new storj.StorageManager(persistence);
var StorageAdapter = require('storj-lib/lib/storage/adapter');
var StorageItem = require('storj-lib/lib/storage/item');
var rstream = manager._storage.createReadStream();
var timestamp = Date.now();
rstream.on('data', function(item) {
rstream.pause();
var total = Object.keys(item.contracts).length;
var endedOrIncomplete = 0;
for (var nodeID in item.contracts) {
var ended = item.contracts[nodeID].get('store_end') < timestamp;
var incomplete = !item.contracts[nodeID].isComplete();
if (ended || incomplete ) {
endedOrIncomplete++;
}
}
if (total === endedOrIncomplete) {
console.log('destroying shard/contract for %s', item.hash);
manager._storage.del(item.hash, function(/* err */) {
rstream.resume();
});
} else {
rstream.resume();
}
});
rstream.on('end', function() {
console.log('flushing shards, some buckets will be inaccessible');
manager._storage.flush(function(err) {
/* istanbul ignore if */
if (err) {
console.log('problem while flushing shards, %s', err.message);
}
console.log('flushing shards finished');
process.exit();
});
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment