Skip to content

Instantly share code, notes, and snippets.

@davidfraser
Created June 10, 2013 15:53
Show Gist options
  • Save davidfraser/5749871 to your computer and use it in GitHub Desktop.
Save davidfraser/5749871 to your computer and use it in GitHub Desktop.
This is a (slightly-cleaned-up) script that I used to restore Firefox history when I'd lost it. Basically give it original.sqlite (the original places database you want to modify), backup.sqlite (an old backup with history that needs to be merged into the original database), and it should generate updated.sqlite which will contain the combinatio…
#!/usr/bin/env python
import sqlite3
import logging
import shutil
# copy
shutil.copy2("original.sqlite", "updated.sqlite")
original = sqlite3.connect("original.sqlite")
backup = sqlite3.connect("backup.sqlite")
updated = sqlite3.connect("updated.sqlite")
logging.getLogger().setLevel(logging.INFO)
def results(c, query):
cursor = c.execute(query)
return cursor.fetchall()
def single_result(c, query):
r = results(c, query)
if len(r) != 1:
raise ValueError("found %d results for %s" % (len(r), query))
return r[0]
def result_dicts(c, query):
cursor = c.execute(query)
fieldnames = [field[0] for field in cursor.description]
r = cursor.fetchall()
return [dict(zip(fieldnames, row)) for row in r]
ignore_diffs = {"id", "favicon_id", "guid", "typed", "hidden"}
update_to_max = {"frecency", "visit_count", "last_visit_date"}
ignore_nones = {"title"}
def transfer_history():
count_total, count_matched, count_exact = 0, 0, 0
count_new_places, count_updated_places, count_new_historyvisits = 0, 0, 0
original_urls = result_dicts(original, "select id, url, guid from moz_places")
original_url_place_map = dict((place["url"], place["id"]) for place in original_urls)
updated_url_place_map = {}
original_place_ids = set(place["id"] for place in original_urls)
available_place_ids = set(i for i in range(min(original_place_ids), max(original_place_ids)) if i not in original_place_ids)
original_historyvisit_ids = set(row[0] for row in results(original, "select id from moz_historyvisits"))
available_historyvisit_ids = set(i for i in range(min(original_historyvisit_ids), max(original_historyvisit_ids)) if i not in original_historyvisit_ids)
for historyvisit in result_dicts(backup, "select * from moz_historyvisits"):
count_total += 1
try:
place = result_dicts(backup, "select * from moz_places where id = %d" % historyvisit["place_id"])[0]
except Exception,e:
logging.warn("Could not find place %d for historyvisit %d", historyvisit["place_id"], historyvisit["id"])
continue
if place["url"] in original_url_place_map:
original_place = result_dicts(original, "select * from moz_places where url = '%s'" % place["url"].replace("'", r"\'"))[0]
count_matched += 1
comb = [(key, (place.get(key, place), original_place.get(key, original_place))) for key in set(place).union(original_place).difference(ignore_diffs)]
dict_diff = dict((key, (ov, mv)) for key, (ov, mv) in comb if ov != mv)
changes = []
for key in update_to_max.intersection(dict_diff):
dict_diff.pop(key)
if place[key] > original_place[key]:
changes.append((key, max(place[key], original_place[key])))
for key in ignore_nones.intersection(dict_diff):
dict_diff.pop(key)
if not original_place[key]:
changes.append((key, place[key]))
new_place_id = original_place["id"]
if changes:
update_sql = "update moz_places set %s where id = :place_id" % (", ".join("%s = :%s" % (key, key) for key, value in changes))
update_dict = dict(changes)
update_dict["place_id"] = new_place_id
updated.execute(update_sql, update_dict)
count_updated_places += 1
if dict_diff:
logging.info("Changes to URL %s: %r", place["url"], dict_diff)
else:
count_exact += 1
del original_place
elif place["url"] in updated_url_place_map:
new_place_id = updated_url_place_map[place["url"]]
else:
new_place_id = available_place_ids.pop() if available_place_ids else max(original_place_ids)+1
updated_place = place.copy()
updated_place["id"] = new_place_id
original_place_ids.add(new_place_id)
updated_url_place_map[updated_place["url"]] = new_place_id
insert_sql = "insert into moz_places(id, url, title, rev_host, visit_count, hidden, typed, favicon_id, frecency, last_visit_date, guid) " + \
"values(:id, :url, :title, :rev_host, :visit_count, :hidden, :typed, :favicon_id, :frecency, :last_visit_date, :guid)"
updated.execute(insert_sql, updated_place)
count_new_places += 1
new_historyvisit = historyvisit.copy()
historyvisit["id"] = new_historyvisit_id = available_historyvisit_ids.pop() if available_historyvisit_ids else max(original_historyvisit_ids)+1
original_historyvisit_ids.add(new_historyvisit_id)
historyvisit["place_id"] = new_place_id
updated.execute("insert into moz_historyvisits(id, from_visit, place_id, visit_date, visit_type, session) " +
"values(:id, :from_visit, :place_id, :visit_date, :visit_type, :session)", historyvisit)
count_new_historyvisits += 1
logging.info("Of %d history entries, %d URLs were already in original, and %d matched exactly", count_total, count_matched, count_exact)
logging.info("Created %d new places, updated %d places, and created %d new history visits", count_new_places, count_updated_places, count_new_historyvisits)
updated.commit()
transfer_history()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment