Get back my books from Kindle

fetch_kindle.js

/* 
 * @fileoverview Program to free the content in kindle books as plain HTML.
 *     
 * This is largely based on reverse engineering kindle cloud app
 * (https://read.amazon.com) to read book data from webSQL.
 * 
 * Access to kindle library is required to download this book.
 */

// The Kindle Compression Module copied from http://read.amazon.com application
// The script reuses the same logic to decompress the fragments
var KindleCompression = function() {
    function h(a, c, g) {
        var f, g = g > 0 ? g : b;
        for (f in c)
            c[f] >= g && (g = c[f] + 1);
        f = g;
        for (var d in a)
            for (var g = a[d], e = 2; e <= g.length; e++) {
                var h = g.substr(0, e);
                c.hasOwnProperty(h) || (c[h] = f++)
            }
        return c
    }
    function j(c, e) {
        var l, h = b;
        l = "";
        for (var r = 0; r < e.length; ) {
            var o = e.charAt(r);
            r++;
            o.charCodeAt(0) <= d ? c.hasOwnProperty(l + o) ? l += o : (l.length > 0 && h < f && (c[l + o] = h,
            h++,
            h === a && (h = g)),
            l = o) : l = ""
        }
        return c
    }
    // This is called when lzExpandWithStaticDictionary doesn't have a dictionary passed in
    // in our case, we always construct the dictionary before decompress
    // function e() {
    //     if (defaultDictionary === void 0 || defaultDictionary === {})
    //         defaultDictionary = j({}, defaultDictionaryString);
    //     return defaultDictionary
    // }
    var d = 9983
      , c = d + 1
      , b = c + 100 + 1
      , f = 65533
      , a = 55295
      , g = 57344;
    return {
        lzCompress: function(k) {
            var e = {}, l = [], h, r = b, o, q, s;
            o = h = "";
            for (var t = 0; t < k.length; ) {
                var u = k.charAt(t);
                t++;
                if (u.charCodeAt(0) <= d) {
                    for (; o.length > 0; ) {
                        q = Math.min(100, o.length);
                        s = o.substr(0, q);
                        o = o.substr(q);
                        l.push(c + q);
                        for (q = 0; q < s.length; q++)
                            l.push(s.charCodeAt(q))
                    }
                    e.hasOwnProperty(h + u) ? h += u : (h.length > 0 && (l.push(h.length === 1 ? h.charCodeAt(0) : e[h]),
                    r < f && (e[h + u] = r,
                    r++,
                    r === a && (r = g))),
                    h = u)
                } else
                    h.length > 0 && (l.push(h.length === 1 ? 
                    h.charCodeAt(0) : e[h]),
                    h = ""),
                    o += u
            }
            for (h.length > 0 && l.push(h.length === 1 ? h.charCodeAt(0) : e[h]); o.length > 0; ) {
                q = Math.min(100, o.length);
                s = o.substr(0, q);
                o = o.substr(q);
                l.push(c + q);
                for (q = 0; q < s.length; q++)
                    l.push(s.charCodeAt(q))
            }
            for (i = 0; i < l.length; i++)
                l[i] = String.fromCharCode(l[i]);
            return l.join("")
        },
        lzExpand: function(k) {
            for (var e = {}, l = [], h, r = b, o = "", q, s = 0; s < k.length; ) {
                h = k.charCodeAt(s);
                s++;
                if (h <= d)
                    h = String.fromCharCode(h);
                else if (h >= b)
                    (h = e[h]) || (h = o + q);
                else {
                    o = h - c;
                    l.push(k.substr(s, o));
                    s += o;
                    o = "";
                    continue
                }
                l.push(h);
                q = h.charAt(0);
                r < f && o.length > 0 && (e[r] = o + q,
                r++,
                r === a && (r = g));
                o = h
            }
            return l.join("")
        },
        lzBuildDictionary: j,
        lzGetDecompressionDictionary: function(a) {
            var b = [], c;
            for (c in a)
                b[a[c]] = c;
            return b
        },
        lzAddStringsToDictionary: h,
        lzAddNumbersToDictionary: function(a, b) {
            for (var c = [], g = 100; g < 1E3; g++)
                c.push("" + g);
            return h(c, a, b)
        },
        lzCompressWithStaticDictionary: function(a, b) {
            if (b === void 0 || b === {})
                b = e();
            var g = [], f, h, o, q;
            h = f = "";
            for (var s = 0; s < a.length; ) {
                var t = a.charAt(s);
                s++;
                if (t.charCodeAt(0) <= d) {
                    for (; h.length > 0; ) {
                        o = Math.min(100, 
                        h.length);
                        q = h.substr(0, o);
                        h = h.substr(o);
                        g.push(c + o);
                        for (o = 0; o < q.length; o++)
                            g.push(q.charCodeAt(o))
                    }
                    b.hasOwnProperty(f + t) ? f += t : (f.length > 0 && g.push(f.length === 1 ? f.charCodeAt(0) : b[f]),
                    f = t)
                } else
                    f.length > 0 && (g.push(f.length === 1 ? f.charCodeAt(0) : b[f]),
                    f = ""),
                    h += t
            }
            for (f.length > 0 && g.push(f.length === 1 ? f.charCodeAt(0) : b[f]); h.length > 0; ) {
                o = Math.min(100, h.length);
                q = h.substr(0, o);
                h = h.substr(o);
                g.push(c + o);
                for (o = 0; o < q.length; o++)
                    g.push(q.charCodeAt(o))
            }
            for (i = 0; i < g.length; i++)
                g[i] = String.fromCharCode(g[i]);
            return g.join("")
        },
        lzExpandWithStaticDictionary: function(a, g, f) {
            // NOTE: g is always defined in our case
            // if (g === void 0 || g === []) {
            //     if (defaultDeDictionary === void 0 || defaultDeDictionary === []) {
            //         e();
            //         defaultDeDictionary = [];
            //         for (var h in defaultDictionary)
            //             defaultDeDictionary[defaultDictionary[h]] = h
            //     }
            //     g = defaultDeDictionary
            // }
            h = d;
            var r = b;
            f !== void 0 && (h = f - 1,
            r = f);
            for (var f = [], o = 0; o < a.length; ) {
                var q = a.charCodeAt(o);
                o++;
                q <= h ? f.push(String.fromCharCode(q)) : q >= r ? f.push(g[q]) : (q -= c,
                f.push(a.substr(o, q)),
                o += q)
            }
            return f.join("")
        }
    }
}()

function s(metadata) { // a is bookinfo.metadata
    var b = {};

    if (metadata.cpr !== void 0) {
        KindleCompression.lzAddStringsToDictionary(metadata.cpr, b),
        KindleCompression.lzAddNumbersToDictionary(b);
        return KindleCompression.lzGetDecompressionDictionary(b);
    }

    if (metadata.cprJson !== void 0) {
        KindleCompression.lzAddStringsToDictionary(metadata.cprJson, b, 256),
        KindleCompression.lzAddNumbersToDictionary(b, 256);
        return KindleCompression.lzGetDecompressionDictionary(b);
    }
}

var fs = require('fs');
var path = require('path');
var sqlite3 = require('sqlite3').verbose();
// 

// http://read.amazon.com stores the ebook with webSQL, which is a sqlite accessible in Chrome
// To locate the sqlite file: http://ahoj.io/how-to-delete-web-sql-database-in-google-chrome
var KINDLE_DB = os.homedir() + '/Library/Application\ Support/Google/Chrome/Default/databases/https_read.amazon.com_0/17';
var db = new sqlite3.Database(KINDLE_DB);

// The following hack is from reverse engineering how kindle cloud app reads data
db.all("select metadata from 'bookinfo'", function(err, rows) {
    rows.forEach(function (row) {
        var metadata = JSON.parse(row.metadata);
        var title = metadata.title;
        var authors = metadata.authorList.join(',');
        // used for dictionary request at https://read.amazon.com/dict/getDefinition?asin=<asin>&word=<word>
        var asin = metadata.asin;
        var ca = s(metadata);

        console.log('staring process book: ' + title);

        var HtmlHeader = '<html><head>' +
            '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">' +
            '<meta name="author" content="' + authors + '">' +
            '</head><body id="' + asin + '">';
        var HtmlFile = path.join(os.tmpdir(), title.replace(/\s+/g, '-') + '.html');

        fs.writeFile(HtmlFile, HtmlHeader);
        console.log("created the file with HTML headers.");

        db.all("select id, piece, other from 'fragments' order by id", function(err, rows) {
            rows.forEach(function (row) {
                var id = row.id;
                var compressedFragmentData = row.piece;
                var uncompressedFragmentData;
                var imageDataMap = JSON.parse(row.other).imageData || {};
                uncompressedFragmentData = KindleCompression.lzExpandWithStaticDictionary(
                    row.piece, ca);
                // replace image path with base64 encoded string
                for (var image in imageDataMap) {
                    uncompressedFragmentData = uncompressedFragmentData.replace(
                        'dataUrl="' + image + '"',
                        'src="' + imageDataMap[image] + '"');
                }
                fs.appendFile(HtmlFile, uncompressedFragmentData);
            });
        });

        fs.appendFile(HtmlFile, '</body></html>');
        console.log("created the file at: " + HtmlFile);
    });
});

hi! thanks for creating this great javascript! I have a question--how does one use it? :)

Fantastic job! I run this script in a debian (jessie) environment. To answer to a previous question node.js is needed to run this script and I had to install the nodejs-osenv package too (maybe a debian related issue).
A first change I did was to include a "require" directive at line 206 to cope with a "os object not found":
var os = require('osenv');
A second change was to modify the name of the function to get the home directory (from homedir to home: maybe a linux issue?) and obviously to change the location of the book database (all the change are at line 214):
var KINDLE_DB = os.home() + "/.config/google-chrome/Default/databases/https_read.amazon.com_0/10"
These changes let the script to run successfully and my books were dumped in html format. However they seemed to be mixed one another. Since the book database is an sqlite one I inspected the schema and I guessed that the query at line 238 should have been modified adding a "where clause":
db.all("select id, piece, other from 'fragments' where asin = '" + asin + "' order by id", function(err, rows) {
This change worked for me and the books are now properly dumped in html format.
I hope this comment could help.
Thanks a lot for this invaluable script!

Thanks to @yangchenyun for the script and to @marcolazzeri for his corrections. It works ;)
A cached version of the webpage suggested by the script author about "how to locate the sqlite database" when using chrome/chromium can be found here: http://archive.is/7wF4E
I have noticed the inner links (chapters, notes) are broken, since they try to call "KindleContentInterface.gotoPosition(6,12399)" and KindleContentInterface is not defined anymore after the conversion.
Any ideas of how to replace all this goToPosition() methods with simple html internal links/anchors during the execution of the script?

Thanks to @yangchenyun for the script and to @marcolazzeri for his corrections from me too! Also thanks to @bitumin for the db link.
I had to change this line:

var KINDLE_DB = os.home() + "/.config/google-chrome/Default/databases/https_read.amazon.com_0/10"
to:
var KINDLE_DB = os.home() + "/.config/google-chrome/Default/databases/https_read.amazon.com_0/75"

I have also made some trivial changes to suit me better and when I get the chance I will work my way through the code to better understand how it works.

Great job extracting the actual encryption functions, that helped me a ton ! goddamn the proprietary apps and crap, i want cold hard pdf :)

i've made a fork to handle internal links and translate the javascript function into actual anchors
here you go @bitumin : https://gist.github.com/EelMood/84140e557065ac3d73f669f120429ae1

with a few tweaks similar to @marcolazzeri suggestions i was able to convert my kindle books from the french cloud reader, under chrome / debian.

thank you all, cheers !

I just had to edit KINDLE_DB, it works like a charm. Using calibre, you can then convert the generated html file to pdf, epub or whatever. Thanks very much!

hi,
This script looks awesome, but i can't manage to use it...
I've installed nodejs and i'm using the command "node fetch_kindle.js" while connected to lire.amazon.fr with the ebook i want to be exported open(on Chrome), But nothing happen.
Can you explain me how to use this script ?

Thanks for this script!
I've made a repo out of it, with small changes:

• added package.json (dependencies)
• configurable file path
• some instructions

https://github.com/d10r/kindle-fetch

You are awesome! This is absolutely brilliant, thanks so much. I hate DRM and am no fan of the Kindle app, or the Kindle, for that matter. The only thing I had to do to get this to work was to run npm install process and change the cont css = multi line variable to this format (my node version must be older):

// improve readability
const css = '\
<style> \
body { \
    margin: 0 auto; \
    max-width: 50em; \
    background: #FFFAFD; \
    font-size:100%; \
    line-height:1.5; \
} \
img { \
    max-width: 100%; \
} \
</style>'

Hello! I've been messing around with this script, and I'm wondering if the db that I have from read.amazon.co.jp are in a slightly different format. I wouldn't think so, but it seems like even though the HTML metadata is being created correctly, and the pages with resource links set up, the images themselves aren't being extracted. If someone could PM me, I'll send over the trouble-causing files as well as the broken HTML result.

Hi,
I am completely lost with this. How do I actually use this ? I have looked at the other versions and yours is the only one that shows 'usage' but where do I put the JS and where do I enter the commands above ?

I was hoping that it would work from Chrome Browser.

I changed the https_read.amazon.com_0/17 to https_read.amazon.com_0/2 as this is the last file in the list.

This is wonderful script. What about CSS styles? Are they stored somewhere inside sqlite database or they are "inlined" into fragments?

should this script still work?

These need to be changed so that the script works on newer versions of Node.js:

line 235
fs.writeFile(HtmlFile, HtmlHeader);
fs.writeFile needs to be changed fs.writeFileSync

line 252
fs.appendFile(HtmlFile, uncompressedFragmentData);
line 256
fs.appendFile(HtmlFile, '</body></html>');
fs.appendFile needs to be changed to fs.appendFileSync

Also, this script can work with Mac Safari as well, just go to line 214:

var KINDLE_DB = os.homedir() + '/Library/Application\ Support/Google/Chrome/Default/databases/https_read.amazon.com_0/17';

change it to:

var KINDLE_DB = '/Users/<user>/Library/Safari/Databases/https_read.amazon.com_0/<db_file>';

The db file name will be a long UID ending in .db.

Make forks instead of describing changes please