Export custom pages from readme.io

export_custom.py

# Python 3

import urllib.parse as p
import bs4
import os
import json
import requests


class ReadMeIOImporterExporter:
    def __init__(self):
        self.cookies = requests.cookies.RequestsCookieJar()
        self.base_url = "https://dash.readme.io/"
        self.xsrf = None
        self.etag = None
        self.login_heads = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Origin': self.base_url,
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'en-US,en;q=0.8',
            'Cache-Control': 'max-age=0',
            'Connection': 'keep-alive',
            'Content-Type': 'application/x-www-form-urlencoded',
            'Host': 'dash.readme.io',
            'Referer': self.base_url + 'login',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
        }
        self.api_heads = {
            "Accept": "application/json, text/plain, */*",
            "Accept-Encoding": "gzip, deflate, sdch",
            "Accept-Language": "en-US,en;q=0.8",
            "Connection": "keep-alive",
            "Host": "dash.readme.io",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36",
            "Referer": self.base_url,
        }
        self.project_url = None
        
    def _update_jar(self, resp):
        for c in resp.cookies:
            self.cookies.set(c.name, c.value)
            if c.name == "XSRF-TOKEN":
                self.xsrf = c.value
        
    def login(self, email, password):
        home = requests.get(self.base_url)
        self._update_jar(home)

        login_req = requests.get(self.base_url+"login", cookies=self.cookies)
        self._update_jar(login_req)

        text = login_req.text
        soup = bs4.BeautifulSoup(text)
        token = soup.find("meta", {"name": "csrf-token"})["content"]

        data = p.urlencode({
                    "email": email,
                    "password": password,
                    "_csrf": token
                }).encode()
        
        self.login_heads["Content-Length"] = len(data)
        sess_req = requests.post(self.base_url + "users/session", data, headers=self.login_heads, cookies=self.cookies)
        self._update_jar(sess_req)
        if sess_req.url == self.base_url:
            return True
        else:
            raise ValueError("Login failed")
        
    def get(self, endpoint, as_json=False):
        if endpoint.startswith("/"):
            endpoint = endpoint[1:]
        req_heads = self.api_heads.copy()
        req_heads["If-None-Match"] = self.etag
        req_heads["X-XSRF-TOKEN"] = self.xsrf + "="
        req = requests.get(self.base_url + endpoint, cookies=self.cookies, headers=req_heads)
        self._update_jar(req)
        if as_json:
            return req.json()
        else:
            return req
        
    def delete(self, endpoint, as_json=False):
        if endpoint.startswith("/"):
            endpoint = endpoint[1:]
        req_heads = self.api_heads.copy()
        req_heads["If-None-Match"] = self.etag
        req_heads["X-XSRF-TOKEN"] = self.xsrf + "="
        req = requests.delete(self.base_url + endpoint, cookies=self.cookies, headers=req_heads)
        self._update_jar(req)
        if as_json:
            return req.json()
        else:
            return req
        
    def post(self, endpoint, data, urlencode=False, as_json=False):
        if endpoint.startswith("/"):
            endpoint = endpoint[1:]
        if urlencode:
            data = p.urlencode(data).encode()
        else:
            data = json.dumps(data)
        req_heads = self.api_heads.copy()
        req_heads["If-None-Match"] = self.etag
        req_heads["X-XSRF-TOKEN"] = self.xsrf + "="
        req_heads["Content-Length"] = len(data)
        req_heads["Content-Type"] = "application/json;charset=UTF-8"
        
        req = requests.post(self.base_url + endpoint, data=data, cookies=self.cookies, headers=req_heads)
        self._update_jar(req)
        if as_json:
            return req.json()
        else:
            return req
        
    def put(self, endpoint, data, urlencode=False, as_json=False):
        if endpoint.startswith("/"):
            endpoint = endpoint[1:]
        if urlencode:
            data = p.urlencode(data).encode()
        else:
            data = json.dumps(data)
        req_heads = self.api_heads.copy()
        req_heads["If-None-Match"] = self.etag
        req_heads["X-XSRF-TOKEN"] = self.xsrf + "="
        req_heads["Content-Length"] = len(data)
        req_heads["Content-Type"] = "application/json;charset=UTF-8"
        
        req = requests.put(self.base_url + endpoint, data=data, cookies=self.cookies, headers=req_heads)
        self._update_jar(req)
        if as_json:
            return req.json()
        else:
            return req
        
    def get_projects(self):
        return self.get("api/projects", as_json=True)
        
    def set_project(self, project_name="my-project"):
        projects = self.get_projects()
        if len(projects) == 1:
            project = projects[0]
        else:
            project = list([x for x in projects if x["name"] == project_name])[0]
        subdomain = project["subdomain"]
        version = project["stable"]["version"]

        project_url = "api/projects/{0}/v{1}/".format(subdomain, version)
        self.project_url = project_url
        
    def get_project(self):
        return self.get(self.project_url + "data").json()
    
    def new_category(self, category_name):
        rec = {"title": category_name}
        return self.post(self.project_url + "category", rec)


READMEIO_UN = "my-email"
READMEIO_PW = "my-password"
OUT_PATH = "/my/output/directory"
PROJECT_NAME = "my-project-name"


rmioi = ReadMeIOImporterExporter()
rmioi.login(READMEIO_UN, READMEIO_PW)
rmioi.set_project()
proj = rmioi.get_project(PROJECT_NAME)

for page in proj["$$custompages"]:
    slug = page["slug"]
    content = page["body"]
    header = "---\ntitle: {0}\nexcerpt: {1}\n---\n".format(
        page.get("title", ""),
        page.get("excerpt", "")
    )
    out_string = header + content
    out_file_name = os.path.join(OUT_PATH, slug + ".md")
    f_out = open(out_file_name, "w")
    f_out.write(out_string)
    f_out.close()