/* | |
* @fileoverview Program to free the content in kindle books as plain HTML. | |
* | |
* This is largely based on reverse engineering kindle cloud app | |
* (https://read.amazon.com) to read book data from webSQL. | |
* | |
* Access to kindle library is required to download this book. | |
*/ | |
// The Kindle Compression Module copied from http://read.amazon.com application | |
// The script reuses the same logic to decompress the fragments | |
var KindleCompression = function() { | |
function h(a, c, g) { | |
var f, g = g > 0 ? g : b; | |
for (f in c) | |
c[f] >= g && (g = c[f] + 1); | |
f = g; | |
for (var d in a) | |
for (var g = a[d], e = 2; e <= g.length; e++) { | |
var h = g.substr(0, e); | |
c.hasOwnProperty(h) || (c[h] = f++) | |
} | |
return c | |
} | |
function j(c, e) { | |
var l, h = b; | |
l = ""; | |
for (var r = 0; r < e.length; ) { | |
var o = e.charAt(r); | |
r++; | |
o.charCodeAt(0) <= d ? c.hasOwnProperty(l + o) ? l += o : (l.length > 0 && h < f && (c[l + o] = h, | |
h++, | |
h === a && (h = g)), | |
l = o) : l = "" | |
} | |
return c | |
} | |
// This is called when lzExpandWithStaticDictionary doesn't have a dictionary passed in | |
// in our case, we always construct the dictionary before decompress | |
// function e() { | |
// if (defaultDictionary === void 0 || defaultDictionary === {}) | |
// defaultDictionary = j({}, defaultDictionaryString); | |
// return defaultDictionary | |
// } | |
var d = 9983 | |
, c = d + 1 | |
, b = c + 100 + 1 | |
, f = 65533 | |
, a = 55295 | |
, g = 57344; | |
return { | |
lzCompress: function(k) { | |
var e = {}, l = [], h, r = b, o, q, s; | |
o = h = ""; | |
for (var t = 0; t < k.length; ) { | |
var u = k.charAt(t); | |
t++; | |
if (u.charCodeAt(0) <= d) { | |
for (; o.length > 0; ) { | |
q = Math.min(100, o.length); | |
s = o.substr(0, q); | |
o = o.substr(q); | |
l.push(c + q); | |
for (q = 0; q < s.length; q++) | |
l.push(s.charCodeAt(q)) | |
} | |
e.hasOwnProperty(h + u) ? h += u : (h.length > 0 && (l.push(h.length === 1 ? h.charCodeAt(0) : e[h]), | |
r < f && (e[h + u] = r, | |
r++, | |
r === a && (r = g))), | |
h = u) | |
} else | |
h.length > 0 && (l.push(h.length === 1 ? | |
h.charCodeAt(0) : e[h]), | |
h = ""), | |
o += u | |
} | |
for (h.length > 0 && l.push(h.length === 1 ? h.charCodeAt(0) : e[h]); o.length > 0; ) { | |
q = Math.min(100, o.length); | |
s = o.substr(0, q); | |
o = o.substr(q); | |
l.push(c + q); | |
for (q = 0; q < s.length; q++) | |
l.push(s.charCodeAt(q)) | |
} | |
for (i = 0; i < l.length; i++) | |
l[i] = String.fromCharCode(l[i]); | |
return l.join("") | |
}, | |
lzExpand: function(k) { | |
for (var e = {}, l = [], h, r = b, o = "", q, s = 0; s < k.length; ) { | |
h = k.charCodeAt(s); | |
s++; | |
if (h <= d) | |
h = String.fromCharCode(h); | |
else if (h >= b) | |
(h = e[h]) || (h = o + q); | |
else { | |
o = h - c; | |
l.push(k.substr(s, o)); | |
s += o; | |
o = ""; | |
continue | |
} | |
l.push(h); | |
q = h.charAt(0); | |
r < f && o.length > 0 && (e[r] = o + q, | |
r++, | |
r === a && (r = g)); | |
o = h | |
} | |
return l.join("") | |
}, | |
lzBuildDictionary: j, | |
lzGetDecompressionDictionary: function(a) { | |
var b = [], c; | |
for (c in a) | |
b[a[c]] = c; | |
return b | |
}, | |
lzAddStringsToDictionary: h, | |
lzAddNumbersToDictionary: function(a, b) { | |
for (var c = [], g = 100; g < 1E3; g++) | |
c.push("" + g); | |
return h(c, a, b) | |
}, | |
lzCompressWithStaticDictionary: function(a, b) { | |
if (b === void 0 || b === {}) | |
b = e(); | |
var g = [], f, h, o, q; | |
h = f = ""; | |
for (var s = 0; s < a.length; ) { | |
var t = a.charAt(s); | |
s++; | |
if (t.charCodeAt(0) <= d) { | |
for (; h.length > 0; ) { | |
o = Math.min(100, | |
h.length); | |
q = h.substr(0, o); | |
h = h.substr(o); | |
g.push(c + o); | |
for (o = 0; o < q.length; o++) | |
g.push(q.charCodeAt(o)) | |
} | |
b.hasOwnProperty(f + t) ? f += t : (f.length > 0 && g.push(f.length === 1 ? f.charCodeAt(0) : b[f]), | |
f = t) | |
} else | |
f.length > 0 && (g.push(f.length === 1 ? f.charCodeAt(0) : b[f]), | |
f = ""), | |
h += t | |
} | |
for (f.length > 0 && g.push(f.length === 1 ? f.charCodeAt(0) : b[f]); h.length > 0; ) { | |
o = Math.min(100, h.length); | |
q = h.substr(0, o); | |
h = h.substr(o); | |
g.push(c + o); | |
for (o = 0; o < q.length; o++) | |
g.push(q.charCodeAt(o)) | |
} | |
for (i = 0; i < g.length; i++) | |
g[i] = String.fromCharCode(g[i]); | |
return g.join("") | |
}, | |
lzExpandWithStaticDictionary: function(a, g, f) { | |
// NOTE: g is always defined in our case | |
// if (g === void 0 || g === []) { | |
// if (defaultDeDictionary === void 0 || defaultDeDictionary === []) { | |
// e(); | |
// defaultDeDictionary = []; | |
// for (var h in defaultDictionary) | |
// defaultDeDictionary[defaultDictionary[h]] = h | |
// } | |
// g = defaultDeDictionary | |
// } | |
h = d; | |
var r = b; | |
f !== void 0 && (h = f - 1, | |
r = f); | |
for (var f = [], o = 0; o < a.length; ) { | |
var q = a.charCodeAt(o); | |
o++; | |
q <= h ? f.push(String.fromCharCode(q)) : q >= r ? f.push(g[q]) : (q -= c, | |
f.push(a.substr(o, q)), | |
o += q) | |
} | |
return f.join("") | |
} | |
} | |
}() | |
function s(metadata) { // a is bookinfo.metadata | |
var b = {}; | |
if (metadata.cpr !== void 0) { | |
KindleCompression.lzAddStringsToDictionary(metadata.cpr, b), | |
KindleCompression.lzAddNumbersToDictionary(b); | |
return KindleCompression.lzGetDecompressionDictionary(b); | |
} | |
if (metadata.cprJson !== void 0) { | |
KindleCompression.lzAddStringsToDictionary(metadata.cprJson, b, 256), | |
KindleCompression.lzAddNumbersToDictionary(b, 256); | |
return KindleCompression.lzGetDecompressionDictionary(b); | |
} | |
} | |
var fs = require('fs'); | |
var path = require('path'); | |
var sqlite3 = require('sqlite3').verbose(); | |
// | |
// http://read.amazon.com stores the ebook with webSQL, which is a sqlite accessible in Chrome | |
// To locate the sqlite file: http://ahoj.io/how-to-delete-web-sql-database-in-google-chrome | |
var KINDLE_DB = os.homedir() + '/Library/Application\ Support/Google/Chrome/Default/databases/https_read.amazon.com_0/17'; | |
var db = new sqlite3.Database(KINDLE_DB); | |
// The following hack is from reverse engineering how kindle cloud app reads data | |
db.all("select metadata from 'bookinfo'", function(err, rows) { | |
rows.forEach(function (row) { | |
var metadata = JSON.parse(row.metadata); | |
var title = metadata.title; | |
var authors = metadata.authorList.join(','); | |
// used for dictionary request at https://read.amazon.com/dict/getDefinition?asin=<asin>&word=<word> | |
var asin = metadata.asin; | |
var ca = s(metadata); | |
console.log('staring process book: ' + title); | |
var HtmlHeader = '<html><head>' + | |
'<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">' + | |
'<meta name="author" content="' + authors + '">' + | |
'</head><body id="' + asin + '">'; | |
var HtmlFile = path.join(os.tmpdir(), title.replace(/\s+/g, '-') + '.html'); | |
fs.writeFile(HtmlFile, HtmlHeader); | |
console.log("created the file with HTML headers."); | |
db.all("select id, piece, other from 'fragments' order by id", function(err, rows) { | |
rows.forEach(function (row) { | |
var id = row.id; | |
var compressedFragmentData = row.piece; | |
var uncompressedFragmentData; | |
var imageDataMap = JSON.parse(row.other).imageData || {}; | |
uncompressedFragmentData = KindleCompression.lzExpandWithStaticDictionary( | |
row.piece, ca); | |
// replace image path with base64 encoded string | |
for (var image in imageDataMap) { | |
uncompressedFragmentData = uncompressedFragmentData.replace( | |
'dataUrl="' + image + '"', | |
'src="' + imageDataMap[image] + '"'); | |
} | |
fs.appendFile(HtmlFile, uncompressedFragmentData); | |
}); | |
}); | |
fs.appendFile(HtmlFile, '</body></html>'); | |
console.log("created the file at: " + HtmlFile); | |
}); | |
}); |
This comment has been minimized.
This comment has been minimized.
Fantastic job! I run this script in a debian (jessie) environment. To answer to a previous question node.js is needed to run this script and I had to install the nodejs-osenv package too (maybe a debian related issue). |
This comment has been minimized.
This comment has been minimized.
Thanks to @yangchenyun for the script and to @marcolazzeri for his corrections. It works ;) |
This comment has been minimized.
This comment has been minimized.
Thanks to @yangchenyun for the script and to @marcolazzeri for his corrections from me too! Also thanks to @bitumin for the db link. var KINDLE_DB = os.home() + "/.config/google-chrome/Default/databases/https_read.amazon.com_0/10" I have also made some trivial changes to suit me better and when I get the chance I will work my way through the code to better understand how it works. |
This comment has been minimized.
This comment has been minimized.
Great job extracting the actual encryption functions, that helped me a ton ! goddamn the proprietary apps and crap, i want cold hard pdf :) i've made a fork to handle internal links and translate the javascript function into actual anchors with a few tweaks similar to @marcolazzeri suggestions i was able to convert my kindle books from the french cloud reader, under chrome / debian. thank you all, cheers ! |
This comment has been minimized.
This comment has been minimized.
I just had to edit |
This comment has been minimized.
This comment has been minimized.
hi, |
This comment has been minimized.
This comment has been minimized.
Thanks for this script!
|
This comment has been minimized.
This comment has been minimized.
You are awesome! This is absolutely brilliant, thanks so much. I hate DRM and am no fan of the Kindle app, or the Kindle, for that matter. The only thing I had to do to get this to work was to run // improve readability const css = '\ <style> \ body { \ margin: 0 auto; \ max-width: 50em; \ background: #FFFAFD; \ font-size:100%; \ line-height:1.5; \ } \ img { \ max-width: 100%; \ } \ </style>' |
This comment has been minimized.
This comment has been minimized.
Hello! I've been messing around with this script, and I'm wondering if the db that I have from read.amazon.co.jp are in a slightly different format. I wouldn't think so, but it seems like even though the HTML metadata is being created correctly, and the pages with resource links set up, the images themselves aren't being extracted. If someone could PM me, I'll send over the trouble-causing files as well as the broken HTML result. |
This comment has been minimized.
This comment has been minimized.
Hi, I was hoping that it would work from Chrome Browser. I changed the https_read.amazon.com_0/17 to https_read.amazon.com_0/2 as this is the last file in the list. |
This comment has been minimized.
This comment has been minimized.
This is wonderful script. What about CSS styles? Are they stored somewhere inside sqlite database or they are "inlined" into fragments? |
This comment has been minimized.
This comment has been minimized.
should this script still work? |
This comment has been minimized.
This comment has been minimized.
These need to be changed so that the script works on newer versions of Node.js: line 235 line 252 Also, this script can work with Mac Safari as well, just go to line 214:
change it to:
The db file name will be a long UID ending in .db. |
This comment has been minimized.
This comment has been minimized.
Make forks instead of describing changes please |
This comment has been minimized.
This comment has been minimized.
I have made the changes suggested by ICWiz above along with a couple of package version updates and have issued a pull request to the https://github.com/d10r/kindle-fetch repository. |
This comment has been minimized.
This comment has been minimized.
Hi guys, For all those wondering if the script still works, it does :) ! Just been using it to convert a book tonight. Cheers ! |
This comment has been minimized.
This comment has been minimized.
This comment has been minimized.
This comment has been minimized.
It doesn't seem to work no more !
After downloading sqlite3 to look at the data base there is no bookinfo (line 274) and the content is pretty empty. It seems the browser doesn't download automatically and I can't manage to right click and ask to download. Seems it has been removed on Amazon Cloud Reader, Here the print witht sqlit3, I checked all the tables :
I would be more than happy to be wrong and it still works ! Cheers, Fiy more details :
|
This comment has been minimized.
hi! thanks for creating this great javascript! I have a question--how does one use it? :)