Skip to content

Instantly share code, notes, and snippets.

@Aymkdn
Created May 18, 2017 08:11
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Aymkdn/153f1da9d172c87c239f4ff15f008530 to your computer and use it in GitHub Desktop.
Save Aymkdn/153f1da9d172c87c239f4ff15f008530 to your computer and use it in GitHub Desktop.
Use node to compare two Sharepoint librairies and copy the source to the destination. It will create the folders and files, but it will also update the Modified date and Modifier author in the destination to match the source
npm install sharepointplus colors
node CompareLibrairies.js "http://my.site.com/my_root/my_source_libary/" "http://my.othersite.com/my_root/my_destination_libary/"
# Limits:
- Don't rename the libraries (the "my_source_library" and "my_destination_library" in the URL should have the same name as the library)
- If you have a dash ('-') in the library name, then you must provide it into the URL
// Command: node index.js "http://my.site.com/my_root/my_source_libary/" "http://my.othersite.com/my_root/my_destination_libary/"
if (process.argv.length !== 4) throw "ERROR: please provide the source and the destination on the command line";
const colors = require('colors');
// load credentials
const credentials = require('../credentials');
const $SP = require('sharepointplus');
const sp = $SP().auth(credentials);
var requestdigest = '';
// contains the references to the items to update in the dest lib
var editorUpdates = new Map();
/*editorUpdates.push({ID:1, Editor:"europe\\olivier_fauveau", Modified:'2017-02-10'});
editorUpdates.push({ID:2, Editor:"americas\\ben_hastings", Modified:'2017-03-20'});*/
const source = {
"url":decodeURIComponent(process.argv[2]).replace(/\/$/,"").split("/").slice(0,-1).join("/")+"/".replace(/\-/g,""),
"library":decodeURIComponent(process.argv[2]).replace(/\/$/,"").split("/").slice(-1).join("")
}
const destination = {
"url":decodeURIComponent(process.argv[3]).replace(/\/$/,"").split("/").slice(0,-1).join("/")+"/".replace(/\-/g,""),
"library":decodeURIComponent(process.argv[3]).replace(/\/$/,"").split("/").slice(-1).join("")
}
const params = {
fields:"ID,FileRef,FSObjType,Created,Modified,Author,Editor",
folderOptions:{show:"FilesAndFolders_Recursive"},
paging:true,
expandUserField:true
}
// list all files from both libraries
let libraries=[
sp.list(source.library, source.url).get(params),
sp.list(destination.library, destination.url).get(params)
]
const lib = {source:new Map(), destination:new Map()};
const missingFiles=new Set(), missingFolders=new Set();
Promise.all(libraries).then(data => {
// transform the data to a Map()
// the FileRef is "XX;#root/site/collection/website/library/folder/file"
// in order to compare it we want to cut at the library level so to get "folder/file"
let treatPath=function(fileRef, obj) {
let url = obj.url + obj.library;
url = url.replace(/\-/g,"").split("/").slice(3).join("/");
return sp.cleanResult(fileRef).slice(url.length);
}
// sort by filename
data.forEach(d => { d.sort((a,b) => a.getAttribute("FileRef").localeCompare(b) )});
// set lib.source and lib.destination
data[0].forEach(row => {
let filepath = treatPath(row.getAttribute("FileRef"), source);
lib.source.set(filepath, row);
});
data[1].forEach(row => {
let filepath = treatPath(row.getAttribute("FileRef"), destination);
lib.destination.set(filepath, row);
});
console.log("Source: ",lib.source.size,"Destination: ",lib.destination.size)
// compare both lib
for (let path of lib.source.keys()) {
if (!lib.destination.has(path)) {
let row = lib.source.get(path);
if (sp.cleanResult(row.getAttribute("FSObjType")) == 1) missingFolders.add(path);
else missingFiles.add(path);
}
}
// we could have "first", "first/second" and "first/second/three"
// in that case we only want to have "first/second/three" to reduce the amount of requests
missingFolders.forEach(folder => {
do {
folder = folder.split("/").slice(0,-1);
if (folder.length > 0) {
folder=folder.join("/");
if (missingFolders.has(folder)) {
missingFolders.delete(folder);
}
}
} while(folder.length>0);
});
console.log(colors.magenta("Missing Folders: "+missingFolders.size))
// we want to keep Author,Created,Editor,Modified info
return libraryReadonly(false);
}).then(() => {
// we first create the folders
let dfd = Promise.resolve();
let res = [...missingFolders].map(function(folder) {
dfd = dfd.then(function() {
console.log("Creating folder « "+folder+" »...")
return sp.list(destination.library, destination.url).createFolder(folder)
});
return dfd
});
return Promise.all(res);
}).then(results => {
var folders = [];
results.forEach(function(res) {
res.forEach(function(folder) {
var key = "/" + folder.BaseName;
var folderDetails = lib.source.get(key);
if (folderDetails && !folder.errorMessage) {
editorUpdates.set(key, {
Key:key,
ID:folder.ID,
Editor:folderDetails.getAttribute("Editor").split(",#")[1].split('|')[1],
Modified:folderDetails.getAttribute("Modified")
});
}
folders.push(folder)
})
})
/*folders.forEach(folder => {
if (folder.errorMessage) console.log(colors.red("=> Folder « "+folder.BaseName+" » hasn't been created: "+folder.errorMessage))
//else console.log("=> Folder "+folder.BaseName+" has been created!")
});*/
console.log(colors.magenta("Missing Files: "+missingFiles.size))
// now we take care of the missing files
const request = require('sp-request').create(credentials);
let dfd = Promise.resolve();
let res = [...missingFiles].map(function(filename) {
dfd = dfd.then(function() {
return new Promise((resolve, reject) => {
let sourceFileUrl = source.url+source.library.replace(/\-/g,"")+"/"+filename;
console.log("Downloading « "+sourceFileUrl+" »...");
request({
url:sourceFileUrl,
method:'GET',
encoding:null
}).then(response => {
if(response.statusCode === 200) {
// encode to b64
let content = base64ArrayBuffer(response.body);
let fileDetails = lib.source.get(filename);
console.log("Creating « " +filename+ " »...")
// upload the file
sp.list(destination.library, destination.url).createFile({
content:content,
encoded:true,
filename:filename,
fields:{
"Created":fileDetails.getAttribute("Created"),
"Modified":fileDetails.getAttribute("Modified")
}
}).then(file => {
// use the "FileRef" as the key
var key = (destination.url.split('/').slice(3).join("/") + destination.library + '/' + filename).replace(/\/\//g,"/");
editorUpdates.set(key, {
Key:key,
Editor:fileDetails.getAttribute("Editor").split(",#")[1].split('|')[1],
Modified:fileDetails.getAttribute("Modified")
});
console.log(colors.green("File « "+file+" » created"));
resolve()
}).catch(error => {
reject(error)
})
} else {
reject("Server returned: "+response.statusCode);
}
});
});
});
return dfd
});
return Promise.all(res);
}).then(() => {
return libraryReadonly(false)
}).then(() => {
// retrieve all files in destination to compare FileRef and find the related ID
return sp.list(destination.library, destination.url).get(params)
}).then(data => {
data.forEach(d => {
// for the files only
if (sp.cleanResult(d.getAttribute("FSObjType")) == 0) {
let key = sp.cleanResult(d.getAttribute("FileRef"));
if (editorUpdates.has(key)) {
let item = editorUpdates.get(key);
item.ID = d.getAttribute("ID");
//console.log("item => ",item)
editorUpdates.set(key, item);
}
}
})
// retrieve digest
return goProm(function(prom_resolve, prom_reject) {
sp.ajax({
method:'POST',
url:destination.url+'_api/contextinfo',
beforeSend: function(xhr) { xhr.setRequestHeader('Accept', 'application/json;odata=verbose'); },
success:function(data) {
prom_resolve(data)
}
})
})
}).then(data => {
requestdigest=(typeof data === "string" ? JSON.parse(data) : data).d.GetContextWebInformation.FormDigestValue;
// now we update Editor for files
var editorUpds=[];
for (let key of editorUpdates.keys()) {
editorUpds.push(editorUpdates.get(key));
}
var dfd = Promise.resolve();
var res = editorUpds.map(function(item) {
dfd = dfd.then(function() {
return updateEditor(item);
});
return dfd
});
Promise.all(res).then(function(res) {
console.log(colors.green("Done!"))
})
}).catch((err) => {
libraryReadonly(false).then(() => {
console.log(colors.red("[ERROR] ",err));
})
})
function libraryReadonly(bool) {
bool = (bool ? 'TRUE' : 'FALSE');
return sp.webService({
service:"Lists",
operation:"UpdateList",
properties:{
listName: destination.library,
updateFields: "<Fields>" +
"<Method ID='1'>" +
'<Field ID="{8c06beca-0777-48f7-91c7-6da68bc07b69}" Name="Created" SourceID="http://schemas.microsoft.com/sharepoint/v3" StaticName="Created" Group="_Hidden" ColName="tp_Created" RowOrdinal="0" ReadOnly="' + bool + '" Type="DateTime" DisplayName="Created" StorageTZ="TRUE"/>' +
"</Method>" +
"<Method ID='2'>" +
'<Field ID="{28cf69c5-fa48-462a-b5cd-27b6f9d2bd5f}" Name="Modified" SourceID="http://schemas.microsoft.com/sharepoint/v3" StaticName="Modified" Group="_Hidden" ColName="tp_Modified" RowOrdinal="0" ReadOnly="' + bool + '" Type="DateTime" DisplayName="Modified" StorageTZ="TRUE"/>' +
"</Method>" +
"</Fields>"
},
webURL:destination.url
})
}
// Encode an ArrayBuffer as a base64 string
// source: https://gist.github.com/jonleighton/958841
function base64ArrayBuffer(arrayBuffer) {
var base64 = ''
var encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
var bytes = new Uint8Array(arrayBuffer)
var byteLength = bytes.byteLength
var byteRemainder = byteLength % 3
var mainLength = byteLength - byteRemainder
var a, b, c, d
var chunk
// Main loop deals with bytes in chunks of 3
for (var i = 0; i < mainLength; i = i + 3) {
// Combine the three bytes into a single integer
chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2]
// Use bitmasks to extract 6-bit segments from the triplet
a = (chunk & 16515072) >> 18 // 16515072 = (2^6 - 1) << 18
b = (chunk & 258048) >> 12 // 258048 = (2^6 - 1) << 12
c = (chunk & 4032) >> 6 // 4032 = (2^6 - 1) << 6
d = chunk & 63 // 63 = 2^6 - 1
// Convert the raw binary segments to the appropriate ASCII encoding
base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d]
}
// Deal with the remaining bytes and padding
if (byteRemainder == 1) {
chunk = bytes[mainLength]
a = (chunk & 252) >> 2 // 252 = (2^6 - 1) << 2
// Set the 4 least significant bits to zero
b = (chunk & 3) << 4 // 3 = 2^2 - 1
base64 += encodings[a] + encodings[b] + '=='
} else if (byteRemainder == 2) {
chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1]
a = (chunk & 64512) >> 10 // 64512 = (2^6 - 1) << 10
b = (chunk & 1008) >> 4 // 1008 = (2^6 - 1) << 4
// Set the 2 least significant bits to zero
c = (chunk & 15) << 2 // 15 = 2^4 - 1
base64 += encodings[a] + encodings[b] + encodings[c] + '='
}
return base64
}
var goProm=function(fct) {
return new Promise(fct)
}
// item = {ID, Editor, Modified}
function updateEditor(item) {
return goProm(function(prom_resolve, prom_reject) {
console.log("Updating editor name for dest file « "+item.Key+" » (#"+item.ID+")...");
var data = '<Request xmlns="http://schemas.microsoft.com/sharepoint/clientquery/2009" SchemaVersion="15.0.0.0" LibraryVersion="15.0.0.0" ApplicationName="Javascript Library"><Actions><ObjectPath Id="1" ObjectPathId="0" /><ObjectPath Id="3" ObjectPathId="2" /><ObjectPath Id="5" ObjectPathId="4" /><ObjectPath Id="7" ObjectPathId="6" /><ObjectIdentityQuery Id="8" ObjectPathId="6" /><ObjectPath Id="10" ObjectPathId="9" /><Method Name="SetFieldValue" Id="11" ObjectPathId="9"><Parameters><Parameter Type="String">Editor</Parameter><Parameter Type="String">-1;#'+item.Editor+'</Parameter></Parameters></Method><Method Name="SetFieldValue" Id="12" ObjectPathId="9"><Parameters><Parameter Type="String">Modified</Parameter><Parameter Type="String">'+item.Modified+'</Parameter></Parameters></Method><Method Name="Update" Id="13" ObjectPathId="9" /><Query Id="14" ObjectPathId="9"><Query SelectAllProperties="false"><Properties><Property Name="Editor" ScalarProperty="true" /><Property Name="Modified" ScalarProperty="true" /></Properties></Query></Query></Actions><ObjectPaths><StaticProperty Id="0" TypeId="{3747adcd-a3c3-41b9-bfab-4a64dd2f1e0a}" Name="Current" /><Property Id="2" ParentId="0" Name="Web" /><Property Id="4" ParentId="2" Name="Lists" /><Method Id="6" ParentId="4" Name="GetByTitle"><Parameters><Parameter Type="String">'+destination.library+'</Parameter></Parameters></Method><Method Id="9" ParentId="6" Name="GetItemById"><Parameters><Parameter Type="Number">'+item.ID+'</Parameter></Parameters></Method></ObjectPaths></Request>';
sp.ajax({
method:'POST',
url:destination.url+'_vti_bin/client.svc/ProcessQuery',
beforeSend: function(xhr) { xhr.setRequestHeader('X-RequestDigest', requestdigest); },
data:data,
success:function(response) {
prom_resolve();
},
error:function(err) {
prom_resolve()
}
})
})
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment