Skip to content

Instantly share code, notes, and snippets.

Forked from pyokagan/
Last active August 29, 2015 14:18
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
What would you like to do?
#! /usr/bin/env python3.2
mozreplcurl - Wrapper around curl that adds on firefox's cookies, and updates firefox's cookie db after curl finishes running.
To update firefox's cookies db when firefox is running, mozrepl is required to be installed.
import sys
import re
from telnetlib import Telnet
from argparse import ArgumentParser
from tempfile import mkstemp
import subprocess
from collections import namedtuple
import os
from urllib.parse import urljoin
from http.cookies import SimpleCookie
2 ways: mozrepl or cookies.sqlite
If we modify cookies.sqlite we use the cookie-jar
option of curl instead.
cookies.sqlite schema
CREATE TABLE moz_cookies (id INTEGER PRIMARY KEY, baseDomain TEXT, name TEXT, value TEXT, host TEXT, path TEXT, expiry INTEGER, lastAccessed INTEGER, creationTime INTEGER, isSecure INTEGER, isHttpOnly INTEGER, CONSTRAINT moz_uniqueid UNIQUE (name, host, path));
CREATE INDEX moz_basedomain ON moz_cookies (baseDomain);
lastAccessed and creationTime is in terms of
PRTime (
PRTime is a 64-bit integer representing the number
of microseconds since the NSPR epoch
a.k.a int(time.time() * (10 ** 6))
class MozRepl(object):
def __init__(self, host="", port=4242): = host
self.port = port
def __enter__(self):
#Start Telnet
self.t = Telnet(, self.port)
#Wait for Prompt, and detect repl name at the same time
_, match, _ = self.t.expect([br'(repl\d*)> '], timeout = 250)
if match is None:
raise Exception('Timeout: Could not detect prompt')
self._context =
self.context =
self.prompt =
return self
def __exit__(self, type, value, traceback):
del self.t
def js(self, command, **kwargs):
#TODO: look for .....> which would mean repl expects a continuation line
#(Then send a ; to abort and hopefully get an error)
from json import dumps
args = sorted([(k, v) for k, v in kwargs.items()])
arglist = ",".join(["repl"] + [x[0] for x in args])
argvalues = ",".join([self.context] + [dumps(x[1]) for x in args])
cmd = "(function({}){{ {} }})({});".format(arglist, command, argvalues)
self.t.write(cmd.encode() + b"\n")
x = self.t.read_until(self.prompt).decode('utf-8', errors = 'replace')
return x[:-(len(self.prompt))]
Cookie = namedtuple("Cookie",
("host", "path", "name", "value", "issecure",
"ishttponly", "issession", "expiry")
host: The host or domain for which the cookie is set.
Presence of a leading dot indicates a domain cookie;
Otherwise, the cookie is treated as a non-domain cookie.
path: Path within the domain for which the cookie is valid.
name: cookie name
value: cookie data
def parse_cookiefile(contents:str) -> [Cookie]:
Each line has the format:
DOMAIN is the domain
NOTSESSIONCOOKIE is "TRUE" if the cookie is not a
session cookie and "FALSE" if the cookie is a
session cookie.
PATH is the root path under the domain where the
cookie is valid. If this is /, the cookie is valid
for the entire domain.
SECUREFLAG is either "TRUE" or "FALSE". Whether or
not a secure connection (HTTPS) is required to read
the cookie.
EXPIRATION is the unixtime where the cookie is set to
NAME is the name of the cookie
VALUE is the cookie value.
Lines starting with # are comments and should
be ignored.
Lines starting with #HttpOnly_ are NOT COMMENTS but
instead denote HTTP-only cookies. The rest
of the line should be parsed as above.
For more info on Http-only cookies see
For more info on the file format see
if isinstance(contents, str):
contents = str.split("\n")
cookies = []
for x in contents:
x = x.strip("\n")
if x.startswith("#HttpOnly_"):
#Httponly cookie
ishttponly = True
x = x[len("#HttpOnly_"):]
elif x.startswith("#") or not x.strip():
#Commented line or blank line
ishttponly = False
host, notsessioncookie, path, issecure, expiry, name, value = x.split("\t", 6)
if notsessioncookie == "TRUE":
issession = False
issession = True
if issecure == "TRUE":
issecure = True
issecure = False
cookies.append(Cookie(host, path, name, value, issecure, ishttponly, issession, int(expiry)))
return cookies
def parse_dumpfile(contents, url):
"""1. Must track the Location redirects!"""
if isinstance(contents, str):
contents = contents.split("\n")
def block(lines):
set_cookies = []
for x in lines:
x = x.strip("\n")
if x.startswith("Date: "):
date = x[len("Date: "):]
for x in lines:
x = x.strip("\n")
if x.startswith("Location: "):
location = x[len("Location: "):]
elif x.startswith("Set-Cookie: "):
set_cookies.append(x[len("Set-Cookie: "):])
return (set_cookies, date, location)
blocks = []
lines = []
cookies = []
for x in contents:
x = x.strip("\n")
if not x:
#New block. Parse previous block
if lines:
lines = []
if lines:
for x in blocks:
cookiestrs, date, location = block(x)
for y in cookiestrs:
cookies.append((y, url, date))
if location:
#Set URL for next block
url = urljoin(url, location)
return cookies
def main(m):
p = ArgumentParser()
#Handle curl's cookie option
p.add_argument("-b", "--cookie", dest="cookie",
action="append", default=[])
#Handle curl's cookie jar option
p.add_argument("-D", "--dump-header", dest="dumpheader",
action="append", default=[])
args, rest = p.parse_known_args()
if rest:
url = rest[-1]
del rest[-1]
p.error("No URL supplied")
after_args = []
_, dump_path = mkstemp()
if ("-c" not in rest) and ("--cookie-jar" not in rest):
#No cookie jar set, so curl will not
#retain cookies on Location redirects.
#Thus, we put in our own dummy cookie jar
_, cj_path = mkstemp()
cj_path = None
#Get cookies from MozRepl
cookies = m.js(r"""
var ios = Components.classes[";1"]
var uri = ios.newURI(url, null, null);
var cookieSvc = Components.classes[";1"]
var cookie = cookieSvc.getCookieStringFromHttp(uri, uri, null);
""", url=url).strip("\n")["curl"] + rest + after_args + ["-b", cookies, "-D", dump_path, url])
#Now read the cookiejar
dump_f = open(dump_path, "r")
dump_contents =
cookies = parse_dumpfile(dump_contents, url)
#Send those cookies to firefox
for x in cookies:
var ios = Components.classes[";1"]
var uri = ios.newURI(url, null, null);
var cookieSvc = Components.classes[";1"]
cookieSvc.setCookieStringFromHttp(uri, uri, null, cookiestr, date, null);
""", url=x[1], cookiestr=x[0], date=x[2]))
#Fetch our own
#If cookiejar is not None, write tempfile
#cookiejar to that cookiejar.
#Remove cookiejar
if args.dumpheader:
f = open(args.dumpheader[-1], "w")
if cj_path:
if __name__ == "__main__":
with MozRepl() as m:
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment