Skip to content

Instantly share code, notes, and snippets.

@bshamric
Last active January 24, 2021 12:08
Show Gist options
  • Star 35 You must be signed in to star a gist
  • Fork 16 You must be signed in to fork a gist
  • Save bshamric/4717583 to your computer and use it in GitHub Desktop.
Save bshamric/4717583 to your computer and use it in GitHub Desktop.
I like phantomjs, but it doesn't directly support getting images from webpages without requesting them separately like in casperjs. I went through QTNetworking code in the phantomjs repo until I figured out where the cache was. To use this, have all three files in the same directory. Then modify test.js for whatever you need. Call phantom js wit…
var fs = require('fs');
//this is the path that QTNetwork classes uses for caching files for it's http client
//the path should be the one that has 16 folders labeled 0,1,2,3,...,F
exports.cachePath = '/path/to/phantomjs/cache/data/folder';
//this is the extension used for files in the cache path
exports.cacheExtension = "d";
//the resources that are to be saved
exports.cachedResources = new Array();
//call this when you first encounter a resource
//includeResource takes the httpResponse from phanomjs and remembers it for later
exports.includeResource = function (httpResponse) {
if(!(httpResponse.url in exports.cachedResources)){
exports.cachedResources[httpResponse.url] = new cachedResource(httpResponse);
}
}
//this takes an httpResponse and builds the object that will be used to get the file contents
function cachedResource (httpResponse) {
this.response = httpResponse;
this.cacheFile = getUrlCacheFilename(this.response.url);
this.cacheFileNoPath = this.cacheFile.clearBefore("/");
this.mimetype = this.response.contentType.clearAfter(";");
this.getContents = function () {
//get all of the contents
var contents = fs.read(exports.cachePath+this.cacheFile,'b');
//get the last header
var lastHeader = this.response.headers[this.response.headers.length-1];
var secondToLastHeader = this.response.headers[this.response.headers.length-2];
var indexOfLastHeader = contents.regexIndexOf(secondToLastHeader.name.escapeRegExp()+".+"+lastHeader.name.escapeRegExp());
//really good chance this will be a problem with non image content types
//I used a hex viewer to open up the QT cache files then came up with this way to find the end of the headers
var lengthOfLastHeaders =
secondToLastHeader.name.length+4+secondToLastHeader.value.length+4+
lastHeader.name.length+4+lastHeader.value.length+1;
return contents.substr(indexOfLastHeader+lengthOfLastHeaders);
}
}
//get the name of the file from the QNetworkDiskCache that phantomjs uses
function getUrlCacheFilename(url,littleEndian,cacheSuffix)
{
//im lazy, so make the last two args optional
littleEndian = typeof littleEndian !== 'undefined' ? littleEndian : true;
cacheSuffix = typeof cacheSuffix !== 'undefined' ? cacheSuffix : '.d';
//convert the sha1 digest to hex
//QT handles caching by converting the SHA1 hash to 64 bit qlonglong
// so the hash gets truncated to 8 bytes
var digest = SHA1(url).substring(0,16).toString();
//if this is a little endian system, reverse the bytes
var littleEndian = true;
if(littleEndian)
{
var result = "";
//the digest is in hex, so take 2 bytes at a time
for (var i = digest.length-1; i > 0; i-=2)
{
result += digest.substr(i-1, 2);
}
digest = result;
}
//convert to js int then to base 36 string, then truncate
var hash = parseInt(digest,16).toString(36).substr(0, 8);
//in QT the filename of a cached url is hash[0]%16 + '/' + hash + '.d'
return (parseInt(hash.charCodeAt(hash.length-1))%16).toString(16) + '/' + hash + cacheSuffix;
}
//get hex SHA1 digest of msg
//borrowed from http://www.webtoolkit.info/javascript-sha1.html
function SHA1(msg) {
function rotate_left(n,s) {
var t4 = ( n<<s ) | (n>>>(32-s));
return t4;
};
function lsb_hex(val) {
var str="";
var i;
var vh;
var vl;
for( i=0; i<=6; i+=2 ) {
vh = (val>>>(i*4+4))&0x0f;
vl = (val>>>(i*4))&0x0f;
str += vh.toString(16) + vl.toString(16);
}
return str;
};
function cvt_hex(val) {
var str="";
var i;
var v;
for( i=7; i>=0; i-- ) {
v = (val>>>(i*4))&0x0f;
str += v.toString(16);
}
return str;
};
function Utf8Encode(string) {
string = string.replace(/\r\n/g,"\n");
var utftext = "";
for (var n = 0; n < string.length; n++) {
var c = string.charCodeAt(n);
if (c < 128) {
utftext += String.fromCharCode(c);
}
else if((c > 127) && (c < 2048)) {
utftext += String.fromCharCode((c >> 6) | 192);
utftext += String.fromCharCode((c & 63) | 128);
}
else {
utftext += String.fromCharCode((c >> 12) | 224);
utftext += String.fromCharCode(((c >> 6) & 63) | 128);
utftext += String.fromCharCode((c & 63) | 128);
}
}
return utftext;
};
var blockstart;
var i, j;
var W = new Array(80);
var H0 = 0x67452301;
var H1 = 0xEFCDAB89;
var H2 = 0x98BADCFE;
var H3 = 0x10325476;
var H4 = 0xC3D2E1F0;
var A, B, C, D, E;
var temp;
msg = Utf8Encode(msg);
var msg_len = msg.length;
var word_array = new Array();
for( i=0; i<msg_len-3; i+=4 ) {
j = msg.charCodeAt(i)<<24 | msg.charCodeAt(i+1)<<16 |
msg.charCodeAt(i+2)<<8 | msg.charCodeAt(i+3);
word_array.push( j );
}
switch( msg_len % 4 ) {
case 0:
i = 0x080000000;
break;
case 1:
i = msg.charCodeAt(msg_len-1)<<24 | 0x0800000;
break;
case 2:
i = msg.charCodeAt(msg_len-2)<<24 | msg.charCodeAt(msg_len-1)<<16 | 0x08000;
break;
case 3:
i = msg.charCodeAt(msg_len-3)<<24 | msg.charCodeAt(msg_len-2)<<16 | msg.charCodeAt(msg_len-1)<<8 | 0x80;
break;
}
word_array.push( i );
while( (word_array.length % 16) != 14 ) word_array.push( 0 );
word_array.push( msg_len>>>29 );
word_array.push( (msg_len<<3)&0x0ffffffff );
for ( blockstart=0; blockstart<word_array.length; blockstart+=16 ) {
for( i=0; i<16; i++ ) W[i] = word_array[blockstart+i];
for( i=16; i<=79; i++ ) W[i] = rotate_left(W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16], 1);
A = H0;
B = H1;
C = H2;
D = H3;
E = H4;
for( i= 0; i<=19; i++ ) {
temp = (rotate_left(A,5) + ((B&C) | (~B&D)) + E + W[i] + 0x5A827999) & 0x0ffffffff;
E = D;
D = C;
C = rotate_left(B,30);
B = A;
A = temp;
}
for( i=20; i<=39; i++ ) {
temp = (rotate_left(A,5) + (B ^ C ^ D) + E + W[i] + 0x6ED9EBA1) & 0x0ffffffff;
E = D;
D = C;
C = rotate_left(B,30);
B = A;
A = temp;
}
for( i=40; i<=59; i++ ) {
temp = (rotate_left(A,5) + ((B&C) | (B&D) | (C&D)) + E + W[i] + 0x8F1BBCDC) & 0x0ffffffff;
E = D;
D = C;
C = rotate_left(B,30);
B = A;
A = temp;
}
for( i=60; i<=79; i++ ) {
temp = (rotate_left(A,5) + (B ^ C ^ D) + E + W[i] + 0xCA62C1D6) & 0x0ffffffff;
E = D;
D = C;
C = rotate_left(B,30);
B = A;
A = temp;
}
H0 = (H0 + A) & 0x0ffffffff;
H1 = (H1 + B) & 0x0ffffffff;
H2 = (H2 + C) & 0x0ffffffff;
H3 = (H3 + D) & 0x0ffffffff;
H4 = (H4 + E) & 0x0ffffffff;
}
var temp = cvt_hex(H0) + cvt_hex(H1) + cvt_hex(H2) + cvt_hex(H3) + cvt_hex(H4);
return temp.toLowerCase();
}
//I'm lazy, this is helpful
String.prototype.clearAfter = function(text) {
var indexOf = this.indexOf(text);
return (indexOf > 0) ? this.substr(0,indexOf) : this.toString();
}
String.prototype.clearBefore = function(text) {
var indexOf = this.indexOf(text);
return (indexOf > 0) ? this.substr(indexOf+1) : this.toString();
}
//"
String.prototype.regexIndexOf = function(regex, startpos) {
var indexOf = this.substring(startpos || 0).search(regex);
return (indexOf >= 0) ? (indexOf + (startpos || 0)) : indexOf;
}
//"
String.prototype.escapeRegExp = function() {
return this.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&").replace(" ","\\s");
}
exports.ext = {
"application/andrew-inset":"ez",
"application/applixware":"aw",
"application/atom+xml":"atom",
"application/atomcat+xml":"atomcat",
"application/atomsvc+xml":"atomsvc",
"application/ccxml+xml":"ccxml",
"application/cdmi-capability":"cdmia",
"application/cdmi-container":"cdmic",
"application/cdmi-domain":"cdmid",
"application/cdmi-object":"cdmio",
"application/cdmi-queue":"cdmiq",
"application/cu-seeme":"cu",
"application/davmount+xml":"davmount",
"application/docbook+xml":"dbk",
"application/dssc+der":"dssc",
"application/dssc+xml":"xdssc",
"application/ecmascript":"ecma",
"application/emma+xml":"emma",
"application/epub+zip":"epub",
"application/exi":"exi",
"application/font-tdpfr":"pfr",
"application/gml+xml":"gml",
"application/gpx+xml":"gpx",
"application/gxf":"gxf",
"application/hyperstudio":"stk",
"application/inkml+xml":"ink",
"application/ipfix":"ipfix",
"application/java-archive":"jar",
"application/java-serialized-object":"ser",
"application/java-vm":"class",
"application/javascript":"js",
"application/json":"json",
"application/jsonml+json":"jsonml",
"application/lost+xml":"lostxml",
"application/mac-binhex40":"hqx",
"application/mac-compactpro":"cpt",
"application/mads+xml":"mads",
"application/marc":"mrc",
"application/marcxml+xml":"mrcx",
"application/mathematica":"ma",
"application/mathml+xml":"mathml",
"application/mbox":"mbox",
"application/mediaservercontrol+xml":"mscml",
"application/metalink+xml":"metalink",
"application/metalink4+xml":"meta4",
"application/mets+xml":"mets",
"application/mods+xml":"mods",
"application/mp21":"m21",
"application/mp4":"mp4s",
"application/msword":"doc",
"application/mxf":"mxf",
"application/octet-stream":"bin",
"application/oda":"oda",
"application/oebps-package+xml":"opf",
"application/ogg":"ogx",
"application/omdoc+xml":"omdoc",
"application/onenote":"onetoc",
"application/oxps":"oxps",
"application/patch-ops-error+xml":"xer",
"application/pdf":"pdf",
"application/pgp-encrypted":"pgp",
"application/pgp-signature":"asc",
"application/pics-rules":"prf",
"application/pkcs10":"p10",
"application/pkcs7-mime":"p7m",
"application/pkcs7-signature":"p7s",
"application/pkcs8":"p8",
"application/pkix-attr-cert":"ac",
"application/pkix-cert":"cer",
"application/pkix-crl":"crl",
"application/pkix-pkipath":"pkipath",
"application/pkixcmp":"pki",
"application/pls+xml":"pls",
"application/postscript":"ai",
"application/prs.cww":"cww",
"application/pskc+xml":"pskcxml",
"application/rdf+xml":"rdf",
"application/reginfo+xml":"rif",
"application/relax-ng-compact-syntax":"rnc",
"application/resource-lists+xml":"rl",
"application/resource-lists-diff+xml":"rld",
"application/rls-services+xml":"rs",
"application/rpki-ghostbusters":"gbr",
"application/rpki-manifest":"mft",
"application/rpki-roa":"roa",
"application/rsd+xml":"rsd",
"application/rss+xml":"rss",
"application/rtf":"rtf",
"application/sbml+xml":"sbml",
"application/scvp-cv-request":"scq",
"application/scvp-cv-response":"scs",
"application/scvp-vp-request":"spq",
"application/scvp-vp-response":"spp",
"application/sdp":"sdp",
"application/set-payment-initiation":"setpay",
"application/set-registration-initiation":"setreg",
"application/shf+xml":"shf",
"application/smil+xml":"smi",
"application/sparql-query":"rq",
"application/sparql-results+xml":"srx",
"application/srgs":"gram",
"application/srgs+xml":"grxml",
"application/sru+xml":"sru",
"application/ssdl+xml":"ssdl",
"application/ssml+xml":"ssml",
"application/tei+xml":"tei",
"application/thraud+xml":"tfi",
"application/timestamped-data":"tsd",
"application/vnd.3gpp.pic-bw-large":"plb",
"application/vnd.3gpp.pic-bw-small":"psb",
"application/vnd.3gpp.pic-bw-var":"pvb",
"application/vnd.3gpp2.tcap":"tcap",
"application/vnd.3m.post-it-notes":"pwn",
"application/vnd.accpac.simply.aso":"aso",
"application/vnd.accpac.simply.imp":"imp",
"application/vnd.acucobol":"acu",
"application/vnd.acucorp":"atc",
"application/vnd.adobe.air-application-installer-package+zip":"air",
"application/vnd.adobe.formscentral.fcdt":"fcdt",
"application/vnd.adobe.fxp":"fxp",
"application/vnd.adobe.xdp+xml":"xdp",
"application/vnd.adobe.xfdf":"xfdf",
"application/vnd.ahead.space":"ahead",
"application/vnd.airzip.filesecure.azf":"azf",
"application/vnd.airzip.filesecure.azs":"azs",
"application/vnd.amazon.ebook":"azw",
"application/vnd.americandynamics.acc":"acc",
"application/vnd.amiga.ami":"ami",
"application/vnd.android.package-archive":"apk",
"application/vnd.anser-web-certificate-issue-initiation":"cii",
"application/vnd.anser-web-funds-transfer-initiation":"fti",
"application/vnd.antix.game-component":"atx",
"application/vnd.apple.installer+xml":"mpkg",
"application/vnd.apple.mpegurl":"m3u8",
"application/vnd.aristanetworks.swi":"swi",
"application/vnd.astraea-software.iota":"iota",
"application/vnd.audiograph":"aep",
"application/vnd.blueice.multipass":"mpm",
"application/vnd.bmi":"bmi",
"application/vnd.businessobjects":"rep",
"application/vnd.chemdraw+xml":"cdxml",
"application/vnd.chipnuts.karaoke-mmd":"mmd",
"application/vnd.cinderella":"cdy",
"application/vnd.claymore":"cla",
"application/vnd.cloanto.rp9":"rp9",
"application/vnd.clonk.c4group":"c4g",
"application/vnd.cluetrust.cartomobile-config":"c11amc",
"application/vnd.cluetrust.cartomobile-config-pkg":"c11amz",
"application/vnd.commonspace":"csp",
"application/vnd.contact.cmsg":"cdbcmsg",
"application/vnd.cosmocaller":"cmc",
"application/vnd.crick.clicker":"clkx",
"application/vnd.crick.clicker.keyboard":"clkk",
"application/vnd.crick.clicker.palette":"clkp",
"application/vnd.crick.clicker.template":"clkt",
"application/vnd.crick.clicker.wordbank":"clkw",
"application/vnd.criticaltools.wbs+xml":"wbs",
"application/vnd.ctc-posml":"pml",
"application/vnd.cups-ppd":"ppd",
"application/vnd.curl.car":"car",
"application/vnd.curl.pcurl":"pcurl",
"application/vnd.dart":"dart",
"application/vnd.data-vision.rdz":"rdz",
"application/vnd.dece.data":"uvf",
"application/vnd.dece.ttml+xml":"uvt",
"application/vnd.dece.unspecified":"uvx",
"application/vnd.dece.zip":"uvz",
"application/vnd.denovo.fcselayout-link":"fe",
"application/vnd.dna":"dna",
"application/vnd.dolby.mlp":"mlp",
"application/vnd.dpgraph":"dpg",
"application/vnd.dreamfactory":"dfac",
"application/vnd.ds-keypoint":"kpxx",
"application/vnd.dvb.ait":"ait",
"application/vnd.dvb.service":"svc",
"application/vnd.dynageo":"geo",
"application/vnd.ecowin.chart":"mag",
"application/vnd.enliven":"nml",
"application/vnd.epson.esf":"esf",
"application/vnd.epson.msf":"msf",
"application/vnd.epson.quickanime":"qam",
"application/vnd.epson.salt":"slt",
"application/vnd.epson.ssf":"ssf",
"application/vnd.eszigno3+xml":"es3",
"application/vnd.ezpix-album":"ez2",
"application/vnd.ezpix-package":"ez3",
"application/vnd.fdf":"fdf",
"application/vnd.fdsn.mseed":"mseed",
"application/vnd.fdsn.seed":"seed",
"application/vnd.flographit":"gph",
"application/vnd.fluxtime.clip":"ftc",
"application/vnd.framemaker":"fm",
"application/vnd.frogans.fnc":"fnc",
"application/vnd.frogans.ltf":"ltf",
"application/vnd.fsc.weblaunch":"fsc",
"application/vnd.fujitsu.oasys":"oas",
"application/vnd.fujitsu.oasys2":"oa2",
"application/vnd.fujitsu.oasys3":"oa3",
"application/vnd.fujitsu.oasysgp":"fg5",
"application/vnd.fujitsu.oasysprs":"bh2",
"application/vnd.fujixerox.ddd":"ddd",
"application/vnd.fujixerox.docuworks":"xdw",
"application/vnd.fujixerox.docuworks.binder":"xbd",
"application/vnd.fuzzysheet":"fzs",
"application/vnd.genomatix.tuxedo":"txd",
"application/vnd.geogebra.file":"ggb",
"application/vnd.geogebra.tool":"ggt",
"application/vnd.geometry-explorer":"gex",
"application/vnd.geonext":"gxt",
"application/vnd.geoplan":"g2w",
"application/vnd.geospace":"g3w",
"application/vnd.gmx":"gmx",
"application/vnd.google-earth.kml+xml":"kml",
"application/vnd.google-earth.kmz":"kmz",
"application/vnd.grafeq":"gqf",
"application/vnd.groove-account":"gac",
"application/vnd.groove-help":"ghf",
"application/vnd.groove-identity-message":"gim",
"application/vnd.groove-injector":"grv",
"application/vnd.groove-tool-message":"gtm",
"application/vnd.groove-tool-template":"tpl",
"application/vnd.groove-vcard":"vcg",
"application/vnd.hal+xml":"hal",
"application/vnd.handheld-entertainment+xml":"zmm",
"application/vnd.hbci":"hbci",
"application/vnd.hhe.lesson-player":"les",
"application/vnd.hp-hpgl":"hpgl",
"application/vnd.hp-hpid":"hpid",
"application/vnd.hp-hps":"hps",
"application/vnd.hp-jlyt":"jlt",
"application/vnd.hp-pcl":"pcl",
"application/vnd.hp-pclxl":"pclxl",
"application/vnd.hydrostatix.sof-data":"sfd",
"application/vnd.ibm.minipay":"mpy",
"application/vnd.ibm.modcap":"afp",
"application/vnd.ibm.rights-management":"irm",
"application/vnd.ibm.secure-container":"sc",
"application/vnd.iccprofile":"icc",
"application/vnd.igloader":"igl",
"application/vnd.immervision-ivp":"ivp",
"application/vnd.immervision-ivu":"ivu",
"application/vnd.insors.igm":"igm",
"application/vnd.intercon.formnet":"xpw",
"application/vnd.intergeo":"i2g",
"application/vnd.intu.qbo":"qbo",
"application/vnd.intu.qfx":"qfx",
"application/vnd.ipunplugged.rcprofile":"rcprofile",
"application/vnd.irepository.package+xml":"irp",
"application/vnd.is-xpr":"xpr",
"application/vnd.isac.fcs":"fcs",
"application/vnd.jam":"jam",
"application/vnd.jcp.javame.midlet-rms":"rms",
"application/vnd.jisp":"jisp",
"application/vnd.joost.joda-archive":"joda",
"application/vnd.kahootz":"ktz",
"application/vnd.kde.karbon":"karbon",
"application/vnd.kde.kchart":"chrt",
"application/vnd.kde.kformula":"kfo",
"application/vnd.kde.kivio":"flw",
"application/vnd.kde.kontour":"kon",
"application/vnd.kde.kpresenter":"kpr",
"application/vnd.kde.kspread":"ksp",
"application/vnd.kde.kword":"kwd",
"application/vnd.kenameaapp":"htke",
"application/vnd.kidspiration":"kia",
"application/vnd.kinar":"kne",
"application/vnd.koan":"skp",
"application/vnd.kodak-descriptor":"sse",
"application/vnd.las.las+xml":"lasxml",
"application/vnd.llamagraphics.life-balance.desktop":"lbd",
"application/vnd.llamagraphics.life-balance.exchange+xml":"lbe",
"application/vnd.lotus-1-2-3":"123",
"application/vnd.lotus-approach":"apr",
"application/vnd.lotus-freelance":"pre",
"application/vnd.lotus-notes":"nsf",
"application/vnd.lotus-organizer":"org",
"application/vnd.lotus-screencam":"scm",
"application/vnd.lotus-wordpro":"lwp",
"application/vnd.macports.portpkg":"portpkg",
"application/vnd.mcd":"mcd",
"application/vnd.medcalcdata":"mc1",
"application/vnd.mediastation.cdkey":"cdkey",
"application/vnd.mfer":"mwf",
"application/vnd.mfmp":"mfm",
"application/vnd.micrografx.flo":"flo",
"application/vnd.micrografx.igx":"igx",
"application/vnd.mif":"mif",
"application/vnd.mobius.daf":"daf",
"application/vnd.mobius.dis":"dis",
"application/vnd.mobius.mbk":"mbk",
"application/vnd.mobius.mqy":"mqy",
"application/vnd.mobius.msl":"msl",
"application/vnd.mobius.plc":"plc",
"application/vnd.mobius.txf":"txf",
"application/vnd.mophun.application":"mpn",
"application/vnd.mophun.certificate":"mpc",
"application/vnd.mozilla.xul+xml":"xul",
"application/vnd.ms-artgalry":"cil",
"application/vnd.ms-cab-compressed":"cab",
"application/vnd.ms-excel":"xls",
"application/vnd.ms-excel.addin.macroenabled.12":"xlam",
"application/vnd.ms-excel.sheet.binary.macroenabled.12":"xlsb",
"application/vnd.ms-excel.sheet.macroenabled.12":"xlsm",
"application/vnd.ms-excel.template.macroenabled.12":"xltm",
"application/vnd.ms-fontobject":"eot",
"application/vnd.ms-htmlhelp":"chm",
"application/vnd.ms-ims":"ims",
"application/vnd.ms-lrm":"lrm",
"application/vnd.ms-officetheme":"thmx",
"application/vnd.ms-pki.seccat":"cat",
"application/vnd.ms-pki.stl":"stl",
"application/vnd.ms-powerpoint":"ppt",
"application/vnd.ms-powerpoint.addin.macroenabled.12":"ppam",
"application/vnd.ms-powerpoint.presentation.macroenabled.12":"pptm",
"application/vnd.ms-powerpoint.slide.macroenabled.12":"sldm",
"application/vnd.ms-powerpoint.slideshow.macroenabled.12":"ppsm",
"application/vnd.ms-powerpoint.template.macroenabled.12":"potm",
"application/vnd.ms-project":"mpp",
"application/vnd.ms-word.document.macroenabled.12":"docm",
"application/vnd.ms-word.template.macroenabled.12":"dotm",
"application/vnd.ms-works":"wps",
"application/vnd.ms-wpl":"wpl",
"application/vnd.ms-xpsdocument":"xps",
"application/vnd.mseq":"mseq",
"application/vnd.musician":"mus",
"application/vnd.muvee.style":"msty",
"application/vnd.mynfc":"taglet",
"application/vnd.neurolanguage.nlu":"nlu",
"application/vnd.nitf":"ntf",
"application/vnd.noblenet-directory":"nnd",
"application/vnd.noblenet-sealer":"nns",
"application/vnd.noblenet-web":"nnw",
"application/vnd.nokia.n-gage.data":"ngdat",
"application/vnd.nokia.n-gage.symbian.install":"n",
"application/vnd.nokia.radio-preset":"rpst",
"application/vnd.nokia.radio-presets":"rpss",
"application/vnd.novadigm.edm":"edm",
"application/vnd.novadigm.edx":"edx",
"application/vnd.novadigm.ext":"ext",
"application/vnd.oasis.opendocument.chart":"odc",
"application/vnd.oasis.opendocument.chart-template":"otc",
"application/vnd.oasis.opendocument.database":"odb",
"application/vnd.oasis.opendocument.formula":"odf",
"application/vnd.oasis.opendocument.formula-template":"odft",
"application/vnd.oasis.opendocument.graphics":"odg",
"application/vnd.oasis.opendocument.graphics-template":"otg",
"application/vnd.oasis.opendocument.image":"odi",
"application/vnd.oasis.opendocument.image-template":"oti",
"application/vnd.oasis.opendocument.presentation":"odp",
"application/vnd.oasis.opendocument.presentation-template":"otp",
"application/vnd.oasis.opendocument.spreadsheet":"ods",
"application/vnd.oasis.opendocument.spreadsheet-template":"ots",
"application/vnd.oasis.opendocument.text":"odt",
"application/vnd.oasis.opendocument.text-master":"odm",
"application/vnd.oasis.opendocument.text-template":"ott",
"application/vnd.oasis.opendocument.text-web":"oth",
"application/vnd.olpc-sugar":"xo",
"application/vnd.oma.dd2+xml":"dd2",
"application/vnd.openofficeorg.extension":"oxt",
"application/vnd.openxmlformats-officedocument.presentationml.presentation":"pptx",
"application/vnd.openxmlformats-officedocument.presentationml.slide":"sldx",
"application/vnd.openxmlformats-officedocument.presentationml.slideshow":"ppsx",
"application/vnd.openxmlformats-officedocument.presentationml.template":"potx",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":"xlsx",
"application/vnd.openxmlformats-officedocument.spreadsheetml.template":"xltx",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document":"docx",
"application/vnd.openxmlformats-officedocument.wordprocessingml.template":"dotx",
"application/vnd.osgeo.mapguide.package":"mgp",
"application/vnd.osgi.dp":"dp",
"application/vnd.osgi.subsystem":"esa",
"application/vnd.palm":"pdb",
"application/vnd.pawaafile":"paw",
"application/vnd.pg.format":"str",
"application/vnd.pg.osasli":"ei6",
"application/vnd.picsel":"efif",
"application/vnd.pmi.widget":"wg",
"application/vnd.pocketlearn":"plf",
"application/vnd.powerbuilder6":"pbd",
"application/vnd.previewsystems.box":"box",
"application/vnd.proteus.magazine":"mgz",
"application/vnd.publishare-delta-tree":"qps",
"application/vnd.pvi.ptid1":"ptid",
"application/vnd.quark.quarkxpress":"qxd",
"application/vnd.realvnc.bed":"bed",
"application/vnd.recordare.musicxml":"mxl",
"application/vnd.recordare.musicxml+xml":"musicxml",
"application/vnd.rig.cryptonote":"cryptonote",
"application/vnd.rim.cod":"cod",
"application/vnd.rn-realmedia":"rm",
"application/vnd.rn-realmedia-vbr":"rmvb",
"application/vnd.route66.link66+xml":"link66",
"application/vnd.sailingtracker.track":"st",
"application/vnd.seemail":"see",
"application/vnd.sema":"sema",
"application/vnd.semd":"semd",
"application/vnd.semf":"semf",
"application/vnd.shana.informed.formdata":"ifm",
"application/vnd.shana.informed.formtemplate":"itp",
"application/vnd.shana.informed.interchange":"iif",
"application/vnd.shana.informed.package":"ipk",
"application/vnd.simtech-mindmapper":"twd",
"application/vnd.smaf":"mmf",
"application/vnd.smart.teacher":"teacher",
"application/vnd.solent.sdkm+xml":"sdkm",
"application/vnd.spotfire.dxp":"dxp",
"application/vnd.spotfire.sfs":"sfs",
"application/vnd.stardivision.calc":"sdc",
"application/vnd.stardivision.draw":"sda",
"application/vnd.stardivision.impress":"sdd",
"application/vnd.stardivision.math":"smf",
"application/vnd.stardivision.writer":"sdw",
"application/vnd.stardivision.writer-global":"sgl",
"application/vnd.stepmania.package":"smzip",
"application/vnd.stepmania.stepchart":"sm",
"application/vnd.sun.xml.calc":"sxc",
"application/vnd.sun.xml.calc.template":"stc",
"application/vnd.sun.xml.draw":"sxd",
"application/vnd.sun.xml.draw.template":"std",
"application/vnd.sun.xml.impress":"sxi",
"application/vnd.sun.xml.impress.template":"sti",
"application/vnd.sun.xml.math":"sxm",
"application/vnd.sun.xml.writer":"sxw",
"application/vnd.sun.xml.writer.global":"sxg",
"application/vnd.sun.xml.writer.template":"stw",
"application/vnd.sus-calendar":"sus",
"application/vnd.svd":"svd",
"application/vnd.symbian.install":"sis",
"application/vnd.syncml+xml":"xsm",
"application/vnd.syncml.dm+wbxml":"bdm",
"application/vnd.syncml.dm+xml":"xdm",
"application/vnd.tao.intent-module-archive":"tao",
"application/vnd.tcpdump.pcap":"pcap",
"application/vnd.tmobile-livetv":"tmo",
"application/vnd.trid.tpt":"tpt",
"application/vnd.triscape.mxs":"mxs",
"application/vnd.trueapp":"tra",
"application/vnd.ufdl":"ufd",
"application/vnd.uiq.theme":"utz",
"application/vnd.umajin":"umj",
"application/vnd.unity":"unityweb",
"application/vnd.uoml+xml":"uoml",
"application/vnd.vcx":"vcx",
"application/vnd.visio":"vsd",
"application/vnd.visionary":"vis",
"application/vnd.vsf":"vsf",
"application/vnd.wap.wbxml":"wbxml",
"application/vnd.wap.wmlc":"wmlc",
"application/vnd.wap.wmlscriptc":"wmlsc",
"application/vnd.webturbo":"wtb",
"application/vnd.wolfram.player":"nbp",
"application/vnd.wordperfect":"wpd",
"application/vnd.wqd":"wqd",
"application/vnd.wt.stf":"stf",
"application/vnd.xara":"xar",
"application/vnd.xfdl":"xfdl",
"application/vnd.yamaha.hv-dic":"hvd",
"application/vnd.yamaha.hv-script":"hvs",
"application/vnd.yamaha.hv-voice":"hvp",
"application/vnd.yamaha.openscoreformat":"osf",
"application/vnd.yamaha.openscoreformat.osfpvg+xml":"osfpvg",
"application/vnd.yamaha.smaf-audio":"saf",
"application/vnd.yamaha.smaf-phrase":"spf",
"application/vnd.yellowriver-custom-menu":"cmp",
"application/vnd.zul":"zir",
"application/vnd.zzazz.deck+xml":"zaz",
"application/voicexml+xml":"vxml",
"application/widget":"wgt",
"application/winhlp":"hlp",
"application/wsdl+xml":"wsdl",
"application/wspolicy+xml":"wspolicy",
"application/x-7z-compressed":"7z",
"application/x-abiword":"abw",
"application/x-ace-compressed":"ace",
"application/x-apple-diskimage":"dmg",
"application/x-authorware-bin":"aab",
"application/x-authorware-map":"aam",
"application/x-authorware-seg":"aas",
"application/x-bcpio":"bcpio",
"application/x-bittorrent":"torrent",
"application/x-blorb":"blb",
"application/x-bzip":"bz",
"application/x-bzip2":"bz2",
"application/x-cbr":"cbr",
"application/x-cdlink":"vcd",
"application/x-cfs-compressed":"cfs",
"application/x-chat":"chat",
"application/x-chess-pgn":"pgn",
"application/x-conference":"nsc",
"application/x-cpio":"cpio",
"application/x-csh":"csh",
"application/x-debian-package":"deb",
"application/x-dgc-compressed":"dgc",
"application/x-director":"dir",
"application/x-doom":"wad",
"application/x-dtbncx+xml":"ncx",
"application/x-dtbook+xml":"dtb",
"application/x-dtbresource+xml":"res",
"application/x-dvi":"dvi",
"application/x-envoy":"evy",
"application/x-eva":"eva",
"application/x-font-bdf":"bdf",
"application/x-font-ghostscript":"gsf",
"application/x-font-linux-psf":"psf",
"application/x-font-otf":"otf",
"application/x-font-pcf":"pcf",
"application/x-font-snf":"snf",
"application/x-font-ttf":"ttf",
"application/x-font-type1":"pfa",
"application/x-font-woff":"woff",
"application/x-freearc":"arc",
"application/x-futuresplash":"spl",
"application/x-gca-compressed":"gca",
"application/x-glulx":"ulx",
"application/x-gnumeric":"gnumeric",
"application/x-gramps-xml":"gramps",
"application/x-gtar":"gtar",
"application/x-hdf":"hdf",
"application/x-install-instructions":"install",
"application/x-iso9660-image":"iso",
"application/x-java-jnlp-file":"jnlp",
"application/x-latex":"latex",
"application/x-lzh-compressed":"lzh",
"application/x-mie":"mie",
"application/x-mobipocket-ebook":"prc",
"application/x-ms-application":"application",
"application/x-ms-shortcut":"lnk",
"application/x-ms-wmd":"wmd",
"application/x-ms-wmz":"wmz",
"application/x-ms-xbap":"xbap",
"application/x-msaccess":"mdb",
"application/x-msbinder":"obd",
"application/x-mscardfile":"crd",
"application/x-msclip":"clp",
"application/x-msdownload":"exe",
"application/x-msmediaview":"mvb",
"application/x-msmetafile":"wmf",
"application/x-msmoney":"mny",
"application/x-mspublisher":"pub",
"application/x-msschedule":"scd",
"application/x-msterminal":"trm",
"application/x-mswrite":"wri",
"application/x-netcdf":"nc",
"application/x-nzb":"nzb",
"application/x-pkcs12":"p12",
"application/x-pkcs7-certificates":"p7b",
"application/x-pkcs7-certreqresp":"p7r",
"application/x-rar-compressed":"rar",
"application/x-research-info-systems":"ris",
"application/x-sh":"sh",
"application/x-shar":"shar",
"application/x-shockwave-flash":"swf",
"application/x-silverlight-app":"xap",
"application/x-sql":"sql",
"application/x-stuffit":"sit",
"application/x-stuffitx":"sitx",
"application/x-subrip":"srt",
"application/x-sv4cpio":"sv4cpio",
"application/x-sv4crc":"sv4crc",
"application/x-t3vm-image":"t3",
"application/x-tads":"gam",
"application/x-tar":"tar",
"application/x-tcl":"tcl",
"application/x-tex":"tex",
"application/x-tex-tfm":"tfm",
"application/x-texinfo":"texinfo",
"application/x-tgif":"obj",
"application/x-ustar":"ustar",
"application/x-wais-source":"src",
"application/x-x509-ca-cert":"der",
"application/x-xfig":"fig",
"application/x-xliff+xml":"xlf",
"application/x-xpinstall":"xpi",
"application/x-xz":"xz",
"application/x-zmachine":"z1",
"application/xaml+xml":"xaml",
"application/xcap-diff+xml":"xdf",
"application/xenc+xml":"xenc",
"application/xhtml+xml":"xhtml",
"application/xml":"xml",
"application/xml-dtd":"dtd",
"application/xop+xml":"xop",
"application/xproc+xml":"xpl",
"application/xslt+xml":"xslt",
"application/xspf+xml":"xspf",
"application/xv+xml":"mxml",
"application/yang":"yang",
"application/yin+xml":"yin",
"application/zip":"zip",
"audio/adpcm":"adp",
"audio/basic":"au",
"audio/midi":"mid",
"audio/mp4":"mp4a",
"audio/mpeg":"mpga",
"audio/ogg":"oga",
"audio/s3m":"s3m",
"audio/silk":"sil",
"audio/vnd.dece.audio":"uva",
"audio/vnd.digital-winds":"eol",
"audio/vnd.dra":"dra",
"audio/vnd.dts":"dts",
"audio/vnd.dts.hd":"dtshd",
"audio/vnd.lucent.voice":"lvp",
"audio/vnd.ms-playready.media.pya":"pya",
"audio/vnd.nuera.ecelp4800":"ecelp4800",
"audio/vnd.nuera.ecelp7470":"ecelp7470",
"audio/vnd.nuera.ecelp9600":"ecelp9600",
"audio/vnd.rip":"rip",
"audio/webm":"weba",
"audio/x-aac":"aac",
"audio/x-aiff":"aif",
"audio/x-caf":"caf",
"audio/x-flac":"flac",
"audio/x-matroska":"mka",
"audio/x-mpegurl":"m3u",
"audio/x-ms-wax":"wax",
"audio/x-ms-wma":"wma",
"audio/x-pn-realaudio":"ram",
"audio/x-pn-realaudio-plugin":"rmp",
"audio/x-wav":"wav",
"audio/xm":"xm",
"chemical/x-cdx":"cdx",
"chemical/x-cif":"cif",
"chemical/x-cmdf":"cmdf",
"chemical/x-cml":"cml",
"chemical/x-csml":"csml",
"chemical/x-xyz":"xyz",
"image/bmp":"bmp",
"image/cgm":"cgm",
"image/g3fax":"g3",
"image/gif":"gif",
"image/ief":"ief",
"image/jpeg":"jpeg",
"image/ktx":"ktx",
"image/png":"png",
"image/prs.btif":"btif",
"image/sgi":"sgi",
"image/svg+xml":"svg",
"image/tiff":"tiff",
"image/vnd.adobe.photoshop":"psd",
"image/vnd.dece.graphic":"uvi",
"image/vnd.dvb.subtitle":"sub",
"image/vnd.djvu":"djvu",
"image/vnd.dwg":"dwg",
"image/vnd.dxf":"dxf",
"image/vnd.fastbidsheet":"fbs",
"image/vnd.fpx":"fpx",
"image/vnd.fst":"fst",
"image/vnd.fujixerox.edmics-mmr":"mmr",
"image/vnd.fujixerox.edmics-rlc":"rlc",
"image/vnd.ms-modi":"mdi",
"image/vnd.ms-photo":"wdp",
"image/vnd.net-fpx":"npx",
"image/vnd.wap.wbmp":"wbmp",
"image/vnd.xiff":"xif",
"image/webp":"webp",
"image/x-3ds":"3ds",
"image/x-cmu-raster":"ras",
"image/x-cmx":"cmx",
"image/x-freehand":"fh",
"image/x-icon":"ico",
"image/x-mrsid-image":"sid",
"image/x-pcx":"pcx",
"image/x-pict":"pic",
"image/x-portable-anymap":"pnm",
"image/x-portable-bitmap":"pbm",
"image/x-portable-graymap":"pgm",
"image/x-portable-pixmap":"ppm",
"image/x-rgb":"rgb",
"image/x-tga":"tga",
"image/x-xbitmap":"xbm",
"image/x-xpixmap":"xpm",
"image/x-xwindowdump":"xwd",
"message/rfc822":"eml",
"model/iges":"igs",
"model/mesh":"msh",
"model/vnd.collada+xml":"dae",
"model/vnd.dwf":"dwf",
"model/vnd.gdl":"gdl",
"model/vnd.gtw":"gtw",
"model/vnd.mts":"mts",
"model/vnd.vtu":"vtu",
"model/vrml":"wrl",
"model/x3d+binary":"x3db",
"model/x3d+vrml":"x3dv",
"model/x3d+xml":"x3d",
"text/cache-manifest":"appcache",
"text/calendar":"ics",
"text/css":"css",
"text/csv":"csv",
"text/javascript":"js",
"text/html":"html",
"text/n3":"n3",
"text/plain":"txt",
"text/prs.lines.tag":"dsc",
"text/richtext":"rtx",
"text/sgml":"sgml",
"text/tab-separated-values":"tsv",
"text/troff":"t",
"text/turtle":"ttl",
"text/uri-list":"uri",
"text/vcard":"vcard",
"text/vnd.curl":"curl",
"text/vnd.curl.dcurl":"dcurl",
"text/vnd.curl.scurl":"scurl",
"text/vnd.curl.mcurl":"mcurl",
"text/vnd.dvb.subtitle":"sub",
"text/vnd.fly":"fly",
"text/vnd.fmi.flexstor":"flx",
"text/vnd.graphviz":"gv",
"text/vnd.in3d.3dml":"3dml",
"text/vnd.in3d.spot":"spot",
"text/vnd.sun.j2me.app-descriptor":"jad",
"text/vnd.wap.wml":"wml",
"text/vnd.wap.wmlscript":"wmls",
"text/x-asm":"s",
"text/x-c":"c",
"text/x-fortran":"f",
"text/x-java-source":"java",
"text/x-opml":"opml",
"text/x-pascal":"p",
"text/x-nfo":"nfo",
"text/x-setext":"etx",
"text/x-sfv":"sfv",
"text/x-uuencode":"uu",
"text/x-vcalendar":"vcs",
"text/x-vcard":"vcf",
"video/3gpp":"3gp",
"video/3gpp2":"3g2",
"video/h261":"h261",
"video/h263":"h263",
"video/h264":"h264",
"video/jpeg":"jpgv",
"video/jpm":"jpm",
"video/mj2":"mj2",
"video/mp4":"mp4",
"video/mpeg":"mpeg",
"video/ogg":"ogv",
"video/quicktime":"qt",
"video/vnd.dece.hd":"uvh",
"video/vnd.dece.mobile":"uvm",
"video/vnd.dece.pd":"uvp",
"video/vnd.dece.sd":"uvs",
"video/vnd.dece.video":"uvv",
"video/vnd.dvb.file":"dvb",
"video/vnd.fvt":"fvt",
"video/vnd.mpegurl":"mxu",
"video/vnd.ms-playready.media.pyv":"pyv",
"video/vnd.uvvu.mp4":"uvu",
"video/vnd.vivo":"viv",
"video/webm":"webm",
"video/x-f4v":"f4v",
"video/x-fli":"fli",
"video/x-flv":"flv",
"video/x-m4v":"m4v",
"video/x-matroska":"mkv",
"video/x-mng":"mng",
"video/x-ms-asf":"asf",
"video/x-ms-vob":"vob",
"video/x-ms-wm":"wm",
"video/x-ms-wmv":"wmv",
"video/x-ms-wmx":"wmx",
"video/x-ms-wvx":"wvx",
"video/x-msvideo":"avi",
"video/x-sgi-movie":"movie",
"video/x-smv":"smv",
"x-conference/x-cooltalk":"ice"
};
//for this to work, you have to call phantomjs with the cache enabled:
//usage: phantomjs --disk-cache=true test.js
var page = require('webpage').create();
var fs = require('fs');
var cache = require('./cache');
var mimetype = require('./mimetype');
//this is the path that QTNetwork classes uses for caching files for it's http client
//the path should be the one that has 16 folders labeled 0,1,2,3,...,F
cache.cachePath = '/Users/brandon/Library/Caches/Ofi Labs/PhantomJS/data7/';
var url = 'http://google.com';
page.viewportSize = { width: 1300, height: 768 };
//when the resource is received, go ahead and include a reference to it in the cache object
page.onResourceReceived = function(response) {
//I only cache images, but you can change this
if(response.contentType.indexOf('image') >= 0)
{
cache.includeResource(response);
}
};
//when the page is done loading, go through each cachedResource and do something with it,
//I'm just saving them to a file
page.onLoadFinished = function(status) {
for(index in cache.cachedResources) {
var file = cache.cachedResources[index].cacheFileNoPath;
var ext = mimetype.ext[cache.cachedResources[index].mimetype];
var finalFile = file.replace("."+cache.cacheExtension,"."+ext);
fs.write('saved/'+finalFile,cache.cachedResources[index].getContents(),'b');
}
};
page.open(url, function () {
page.render('saved/google.pdf');
phantom.exit();
});
@OllieJones
Copy link

This is great. Thanks for working out how to read these cache files!

I needed to handle more types of content files (mostly pdf). That required tightening up the interpretation of the header stuff in the cache files. Here is my fork of your gist. https://gist.github.com/OllieJones/b435641f610ff6f2bb5e

Please do whatever you wish with it. Thanks again.

@svarvel
Copy link

svarvel commented Sep 23, 2016

Hello, thank you for this. I gave it a try but I am seeing issues. Is it possible that phantomjs changed the way they encode on disk? I checked your code and saw that (for example) you skip the first 8 bytes, then read 4 bytes for the URL length, then read the URL (some sort of TLV encoding?); this already fails for me. Can you point me to where phantomjs does such encoding so I/we can try to update the code?

@mike-aungsan
Copy link

Hi,

This line has an issue. Cache.js:29
var contents = fs.read(exports.cachePath+this.cacheFile,'b');

TypeError: "size" argument must be a number
at Function.Buffer.allocUnsafe (buffer.js:161:3)
at Object.fs.read (fs.js:579:21)

Any help?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment