Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Get back my books from Kindle
/*
* @fileoverview Program to free the content in kindle books as plain HTML.
*
* This is largely based on reverse engineering kindle cloud app
* (https://read.amazon.com) to read book data from webSQL.
*
* Access to kindle library is required to download this book.
*/
// The Kindle Compression Module copied from http://read.amazon.com application
// The script reuses the same logic to decompress the fragments
var KindleCompression = function() {
function h(a, c, g) {
var f, g = g > 0 ? g : b;
for (f in c)
c[f] >= g && (g = c[f] + 1);
f = g;
for (var d in a)
for (var g = a[d], e = 2; e <= g.length; e++) {
var h = g.substr(0, e);
c.hasOwnProperty(h) || (c[h] = f++)
}
return c
}
function j(c, e) {
var l, h = b;
l = "";
for (var r = 0; r < e.length; ) {
var o = e.charAt(r);
r++;
o.charCodeAt(0) <= d ? c.hasOwnProperty(l + o) ? l += o : (l.length > 0 && h < f && (c[l + o] = h,
h++,
h === a && (h = g)),
l = o) : l = ""
}
return c
}
// This is called when lzExpandWithStaticDictionary doesn't have a dictionary passed in
// in our case, we always construct the dictionary before decompress
// function e() {
// if (defaultDictionary === void 0 || defaultDictionary === {})
// defaultDictionary = j({}, defaultDictionaryString);
// return defaultDictionary
// }
var d = 9983
, c = d + 1
, b = c + 100 + 1
, f = 65533
, a = 55295
, g = 57344;
return {
lzCompress: function(k) {
var e = {}, l = [], h, r = b, o, q, s;
o = h = "";
for (var t = 0; t < k.length; ) {
var u = k.charAt(t);
t++;
if (u.charCodeAt(0) <= d) {
for (; o.length > 0; ) {
q = Math.min(100, o.length);
s = o.substr(0, q);
o = o.substr(q);
l.push(c + q);
for (q = 0; q < s.length; q++)
l.push(s.charCodeAt(q))
}
e.hasOwnProperty(h + u) ? h += u : (h.length > 0 && (l.push(h.length === 1 ? h.charCodeAt(0) : e[h]),
r < f && (e[h + u] = r,
r++,
r === a && (r = g))),
h = u)
} else
h.length > 0 && (l.push(h.length === 1 ?
h.charCodeAt(0) : e[h]),
h = ""),
o += u
}
for (h.length > 0 && l.push(h.length === 1 ? h.charCodeAt(0) : e[h]); o.length > 0; ) {
q = Math.min(100, o.length);
s = o.substr(0, q);
o = o.substr(q);
l.push(c + q);
for (q = 0; q < s.length; q++)
l.push(s.charCodeAt(q))
}
for (i = 0; i < l.length; i++)
l[i] = String.fromCharCode(l[i]);
return l.join("")
},
lzExpand: function(k) {
for (var e = {}, l = [], h, r = b, o = "", q, s = 0; s < k.length; ) {
h = k.charCodeAt(s);
s++;
if (h <= d)
h = String.fromCharCode(h);
else if (h >= b)
(h = e[h]) || (h = o + q);
else {
o = h - c;
l.push(k.substr(s, o));
s += o;
o = "";
continue
}
l.push(h);
q = h.charAt(0);
r < f && o.length > 0 && (e[r] = o + q,
r++,
r === a && (r = g));
o = h
}
return l.join("")
},
lzBuildDictionary: j,
lzGetDecompressionDictionary: function(a) {
var b = [], c;
for (c in a)
b[a[c]] = c;
return b
},
lzAddStringsToDictionary: h,
lzAddNumbersToDictionary: function(a, b) {
for (var c = [], g = 100; g < 1E3; g++)
c.push("" + g);
return h(c, a, b)
},
lzCompressWithStaticDictionary: function(a, b) {
if (b === void 0 || b === {})
b = e();
var g = [], f, h, o, q;
h = f = "";
for (var s = 0; s < a.length; ) {
var t = a.charAt(s);
s++;
if (t.charCodeAt(0) <= d) {
for (; h.length > 0; ) {
o = Math.min(100,
h.length);
q = h.substr(0, o);
h = h.substr(o);
g.push(c + o);
for (o = 0; o < q.length; o++)
g.push(q.charCodeAt(o))
}
b.hasOwnProperty(f + t) ? f += t : (f.length > 0 && g.push(f.length === 1 ? f.charCodeAt(0) : b[f]),
f = t)
} else
f.length > 0 && (g.push(f.length === 1 ? f.charCodeAt(0) : b[f]),
f = ""),
h += t
}
for (f.length > 0 && g.push(f.length === 1 ? f.charCodeAt(0) : b[f]); h.length > 0; ) {
o = Math.min(100, h.length);
q = h.substr(0, o);
h = h.substr(o);
g.push(c + o);
for (o = 0; o < q.length; o++)
g.push(q.charCodeAt(o))
}
for (i = 0; i < g.length; i++)
g[i] = String.fromCharCode(g[i]);
return g.join("")
},
lzExpandWithStaticDictionary: function(a, g, f) {
// NOTE: g is always defined in our case
// if (g === void 0 || g === []) {
// if (defaultDeDictionary === void 0 || defaultDeDictionary === []) {
// e();
// defaultDeDictionary = [];
// for (var h in defaultDictionary)
// defaultDeDictionary[defaultDictionary[h]] = h
// }
// g = defaultDeDictionary
// }
h = d;
var r = b;
f !== void 0 && (h = f - 1,
r = f);
for (var f = [], o = 0; o < a.length; ) {
var q = a.charCodeAt(o);
o++;
q <= h ? f.push(String.fromCharCode(q)) : q >= r ? f.push(g[q]) : (q -= c,
f.push(a.substr(o, q)),
o += q)
}
return f.join("")
}
}
}()
function s(metadata) { // a is bookinfo.metadata
var b = {};
if (metadata.cpr !== void 0) {
KindleCompression.lzAddStringsToDictionary(metadata.cpr, b),
KindleCompression.lzAddNumbersToDictionary(b);
return KindleCompression.lzGetDecompressionDictionary(b);
}
if (metadata.cprJson !== void 0) {
KindleCompression.lzAddStringsToDictionary(metadata.cprJson, b, 256),
KindleCompression.lzAddNumbersToDictionary(b, 256);
return KindleCompression.lzGetDecompressionDictionary(b);
}
}
var fs = require('fs');
var path = require('path');
var sqlite3 = require('sqlite3').verbose();
//
// http://read.amazon.com stores the ebook with webSQL, which is a sqlite accessible in Chrome
// To locate the sqlite file: http://ahoj.io/how-to-delete-web-sql-database-in-google-chrome
var KINDLE_DB = os.homedir() + '/Library/Application\ Support/Google/Chrome/Default/databases/https_read.amazon.com_0/17';
var db = new sqlite3.Database(KINDLE_DB);
// The following hack is from reverse engineering how kindle cloud app reads data
db.all("select metadata from 'bookinfo'", function(err, rows) {
rows.forEach(function (row) {
var metadata = JSON.parse(row.metadata);
var title = metadata.title;
var authors = metadata.authorList.join(',');
// used for dictionary request at https://read.amazon.com/dict/getDefinition?asin=<asin>&word=<word>
var asin = metadata.asin;
var ca = s(metadata);
console.log('staring process book: ' + title);
var HtmlHeader = '<html><head>' +
'<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">' +
'<meta name="author" content="' + authors + '">' +
'</head><body id="' + asin + '">';
var HtmlFile = path.join(os.tmpdir(), title.replace(/\s+/g, '-') + '.html');
fs.writeFile(HtmlFile, HtmlHeader);
console.log("created the file with HTML headers.");
db.all("select id, piece, other from 'fragments' order by id", function(err, rows) {
rows.forEach(function (row) {
var id = row.id;
var compressedFragmentData = row.piece;
var uncompressedFragmentData;
var imageDataMap = JSON.parse(row.other).imageData || {};
uncompressedFragmentData = KindleCompression.lzExpandWithStaticDictionary(
row.piece, ca);
// replace image path with base64 encoded string
for (var image in imageDataMap) {
uncompressedFragmentData = uncompressedFragmentData.replace(
'dataUrl="' + image + '"',
'src="' + imageDataMap[image] + '"');
}
fs.appendFile(HtmlFile, uncompressedFragmentData);
});
});
fs.appendFile(HtmlFile, '</body></html>');
console.log("created the file at: " + HtmlFile);
});
});
@indychris33

This comment has been minimized.

Copy link

@indychris33 indychris33 commented Jun 24, 2016

hi! thanks for creating this great javascript! I have a question--how does one use it? :)

@marcolazzeri

This comment has been minimized.

Copy link

@marcolazzeri marcolazzeri commented Jul 27, 2016

Fantastic job! I run this script in a debian (jessie) environment. To answer to a previous question node.js is needed to run this script and I had to install the nodejs-osenv package too (maybe a debian related issue).
A first change I did was to include a "require" directive at line 206 to cope with a "os object not found":
var os = require('osenv');
A second change was to modify the name of the function to get the home directory (from homedir to home: maybe a linux issue?) and obviously to change the location of the book database (all the change are at line 214):
var KINDLE_DB = os.home() + "/.config/google-chrome/Default/databases/https_read.amazon.com_0/10"
These changes let the script to run successfully and my books were dumped in html format. However they seemed to be mixed one another. Since the book database is an sqlite one I inspected the schema and I guessed that the query at line 238 should have been modified adding a "where clause":
db.all("select id, piece, other from 'fragments' where asin = '" + asin + "' order by id", function(err, rows) {
This change worked for me and the books are now properly dumped in html format.
I hope this comment could help.
Thanks a lot for this invaluable script!

@bitumin

This comment has been minimized.

Copy link

@bitumin bitumin commented Aug 15, 2016

Thanks to @yangchenyun for the script and to @marcolazzeri for his corrections. It works ;)
A cached version of the webpage suggested by the script author about "how to locate the sqlite database" when using chrome/chromium can be found here: http://archive.is/7wF4E
I have noticed the inner links (chapters, notes) are broken, since they try to call "KindleContentInterface.gotoPosition(6,12399)" and KindleContentInterface is not defined anymore after the conversion.
Any ideas of how to replace all this goToPosition() methods with simple html internal links/anchors during the execution of the script?

@philiprhoades

This comment has been minimized.

Copy link

@philiprhoades philiprhoades commented Sep 18, 2016

Thanks to @yangchenyun for the script and to @marcolazzeri for his corrections from me too! Also thanks to @bitumin for the db link.
I had to change this line:

var KINDLE_DB = os.home() + "/.config/google-chrome/Default/databases/https_read.amazon.com_0/10"
to:
var KINDLE_DB = os.home() + "/.config/google-chrome/Default/databases/https_read.amazon.com_0/75"

I have also made some trivial changes to suit me better and when I get the chance I will work my way through the code to better understand how it works.

@moodule

This comment has been minimized.

Copy link

@moodule moodule commented Oct 2, 2016

Great job extracting the actual encryption functions, that helped me a ton ! goddamn the proprietary apps and crap, i want cold hard pdf :)

i've made a fork to handle internal links and translate the javascript function into actual anchors
here you go @bitumin : https://gist.github.com/EelMood/84140e557065ac3d73f669f120429ae1

with a few tweaks similar to @marcolazzeri suggestions i was able to convert my kindle books from the french cloud reader, under chrome / debian.

thank you all, cheers !

@tokland

This comment has been minimized.

Copy link

@tokland tokland commented Nov 13, 2016

I just had to edit KINDLE_DB, it works like a charm. Using calibre, you can then convert the generated html file to pdf, epub or whatever. Thanks very much!

@barthelemypousset

This comment has been minimized.

Copy link

@barthelemypousset barthelemypousset commented Dec 3, 2016

hi,
This script looks awesome, but i can't manage to use it...
I've installed nodejs and i'm using the command "node fetch_kindle.js" while connected to lire.amazon.fr with the ebook i want to be exported open(on Chrome), But nothing happen.
Can you explain me how to use this script ?

@d10r

This comment has been minimized.

Copy link

@d10r d10r commented Jan 26, 2017

Thanks for this script!
I've made a repo out of it, with small changes:

  • added package.json (dependencies)
  • configurable file path
  • some instructions

https://github.com/d10r/kindle-fetch

@darkerego

This comment has been minimized.

Copy link

@darkerego darkerego commented Feb 23, 2017

You are awesome! This is absolutely brilliant, thanks so much. I hate DRM and am no fan of the Kindle app, or the Kindle, for that matter. The only thing I had to do to get this to work was to run npm install process and change the cont css = multi line variable to this format (my node version must be older):

// improve readability
const css = '\
<style> \
body { \
    margin: 0 auto; \
    max-width: 50em; \
    background: #FFFAFD; \
    font-size:100%; \
    line-height:1.5; \
} \
img { \
    max-width: 100%; \
} \
</style>'
@SignificantOtter

This comment has been minimized.

Copy link

@SignificantOtter SignificantOtter commented Feb 27, 2017

Hello! I've been messing around with this script, and I'm wondering if the db that I have from read.amazon.co.jp are in a slightly different format. I wouldn't think so, but it seems like even though the HTML metadata is being created correctly, and the pages with resource links set up, the images themselves aren't being extracted. If someone could PM me, I'll send over the trouble-causing files as well as the broken HTML result.

@jasonc310771

This comment has been minimized.

Copy link

@jasonc310771 jasonc310771 commented Jul 3, 2017

Hi,
I am completely lost with this. How do I actually use this ? I have looked at the other versions and yours is the only one that shows 'usage' but where do I put the JS and where do I enter the commands above ?

I was hoping that it would work from Chrome Browser.

I changed the https_read.amazon.com_0/17 to https_read.amazon.com_0/2 as this is the last file in the list.

@HeathenGirl

This comment has been minimized.

Copy link

@HeathenGirl HeathenGirl commented Aug 25, 2017

This is wonderful script. What about CSS styles? Are they stored somewhere inside sqlite database or they are "inlined" into fragments?

@benjaminadrom

This comment has been minimized.

Copy link

@benjaminadrom benjaminadrom commented Apr 2, 2019

should this script still work?

@ICWiz

This comment has been minimized.

Copy link

@ICWiz ICWiz commented Apr 5, 2019

These need to be changed so that the script works on newer versions of Node.js:

line 235
fs.writeFile(HtmlFile, HtmlHeader);
fs.writeFile needs to be changed fs.writeFileSync

line 252
fs.appendFile(HtmlFile, uncompressedFragmentData);
line 256
fs.appendFile(HtmlFile, '</body></html>');
fs.appendFile needs to be changed to fs.appendFileSync

Also, this script can work with Mac Safari as well, just go to line 214:

var KINDLE_DB = os.homedir() + '/Library/Application\ Support/Google/Chrome/Default/databases/https_read.amazon.com_0/17';

change it to:

var KINDLE_DB = '/Users/<user>/Library/Safari/Databases/https_read.amazon.com_0/<db_file>';

The db file name will be a long UID ending in .db.

@glachancecmaisonneuve

This comment has been minimized.

Copy link

@glachancecmaisonneuve glachancecmaisonneuve commented Jun 9, 2019

Make forks instead of describing changes please

@varun-ramraj

This comment has been minimized.

Copy link

@varun-ramraj varun-ramraj commented May 2, 2020

I have made the changes suggested by ICWiz above along with a couple of package version updates and have issued a pull request to the https://github.com/d10r/kindle-fetch repository.

@iongradea

This comment has been minimized.

Copy link

@iongradea iongradea commented May 29, 2020

Hi guys,

For all those wondering if the script still works, it does :) ! Just been using it to convert a book tonight.
Have a look at the different version for node, chrome and macOS I have on this issue : d10r/kindle-fetch#9.
Thx to all the people that did that amazing work ! Been struggling for several week with a reliable and free way to crack DRM (hate DRM) !!!

Cheers !

@dineshkoravi

This comment has been minimized.

Copy link

@dineshkoravi dineshkoravi commented May 30, 2020

I tried to convert a file. The output file is 0KB. please help me.

image

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.